00000080:00200000:1.0:1775764495.412862:0:51114:0:(rw26.c:916:ll_write_end()) page@000000003373f2e1[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.412865:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000005452f4ac 17ffffc0001021 2:0 ffff8b23b9f4d1c0 688 lru 00000080:00200000:1.0:1775764495.412871:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000dd1bb50b 688: 1< 0 - - > 2< 2818048 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.412873:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000003373f2e1 00000080:00200000:1.0:1775764495.412874:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 177. 00000080:00200000:1.0:1775764495.412876:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 689 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.412895:0:51114:0:(rw26.c:858:ll_write_end()) pos 2822144, len 4096, copied 4096 00000080:00200000:1.0:1775764495.412899:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000b899e827[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.412902:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @0000000027e034d3 17ffffc0001001 3:0 ffff8b23b9f4d540 689 lru 00000080:00200000:1.0:1775764495.412907:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000656e0dce 689: 1< 0 - - > 2< 2822144 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.412909:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000b899e827 00000080:00200000:1.0:1775764495.412911:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 178. 00000080:00200000:1.0:1775764495.412913:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 690 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.412924:0:51114:0:(rw26.c:858:ll_write_end()) pos 2826240, len 4096, copied 4096 00000080:00200000:1.0:1775764495.412927:0:51114:0:(rw26.c:916:ll_write_end()) page@000000004ba6f2bc[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.412931:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000cd181af2 17ffffc0001001 3:0 ffff8b23b9f4d700 690 lru 00000080:00200000:1.0:1775764495.412937:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@0000000039b4eaaa 690: 1< 0 - - > 2< 2826240 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.412939:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000004ba6f2bc 00000080:00200000:1.0:1775764495.412940:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 179. 00000080:00200000:1.0:1775764495.412941:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 691 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.412951:0:51114:0:(rw26.c:858:ll_write_end()) pos 2830336, len 4096, copied 4096 00000080:00200000:1.0:1775764495.412954:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000d234c034[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.412957:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000004b6abe8c 17ffffc0001001 3:0 ffff8b23b9f4d7e0 691 lru 00000080:00200000:1.0:1775764495.412963:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000000025ec0d 691: 1< 0 - - > 2< 2830336 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.412965:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000d234c034 00000080:00200000:1.0:1775764495.412966:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 180. 00000080:00200000:1.0:1775764495.412968:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 692 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.412977:0:51114:0:(rw26.c:858:ll_write_end()) pos 2834432, len 4096, copied 4096 00000080:00200000:1.0:1775764495.412981:0:51114:0:(rw26.c:916:ll_write_end()) page@000000009a52feb8[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.412984:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000004b4c9036 17ffffc0001001 3:0 ffff8b23b9f4dd20 692 lru 00000080:00200000:1.0:1775764495.412990:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000d430c471 692: 1< 0 - - > 2< 2834432 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.412992:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000009a52feb8 00000080:00200000:1.0:1775764495.412993:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 181. 00000080:00200000:1.0:1775764495.412995:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 693 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413005:0:51114:0:(rw26.c:858:ll_write_end()) pos 2838528, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413009:0:51114:0:(rw26.c:916:ll_write_end()) page@0000000035adbbcb[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413012:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000c326edcb 17ffffc0001001 3:0 ffff8b23b9f4d380 693 lru 00000080:00200000:1.0:1775764495.413018:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000b7e254d3 693: 1< 0 - - > 2< 2838528 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413020:0:51114:0:(rw26.c:916:ll_write_end()) end page@0000000035adbbcb 00000080:00200000:1.0:1775764495.413021:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 182. 00000080:00200000:1.0:1775764495.413022:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 694 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413037:0:51114:0:(rw26.c:858:ll_write_end()) pos 2842624, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413041:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000de01b40d[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413043:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000158f8c48 17ffffc0001001 3:0 ffff8b23b9f4d460 694 lru 00000080:00200000:1.0:1775764495.413048:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000631404c9 694: 1< 0 - - > 2< 2842624 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413050:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000de01b40d 00000080:00200000:1.0:1775764495.413051:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 183. 00000080:00200000:1.0:1775764495.413053:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 695 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413077:0:51114:0:(rw26.c:858:ll_write_end()) pos 2846720, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413081:0:51114:0:(rw26.c:916:ll_write_end()) page@000000007de8a189[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413084:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000001d2a124c 17ffffc0001001 3:0 ffff8b23b9f4d0e0 695 lru 00000080:00200000:1.0:1775764495.413089:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000002f9529d1 695: 1< 0 - - > 2< 2846720 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413090:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000007de8a189 00000080:00200000:1.0:1775764495.413091:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 184. 00000080:00200000:1.0:1775764495.413093:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 696 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413107:0:51114:0:(rw26.c:858:ll_write_end()) pos 2850816, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413111:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000583d85ab[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413113:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000c4264075 17ffffc0001001 3:0 ffff8b23b9f4d2a0 696 lru 00000080:00200000:1.0:1775764495.413118:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@0000000079bc4202 696: 1< 0 - - > 2< 2850816 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413120:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000583d85ab 00000080:00200000:1.0:1775764495.413121:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 185. 00000080:00200000:1.0:1775764495.413122:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 697 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413134:0:51114:0:(rw26.c:858:ll_write_end()) pos 2854912, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413137:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000071276d9[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413140:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000004b8862cb 17ffffc0001001 3:0 ffff8b23b9f4d8c0 697 lru 00000080:00200000:1.0:1775764495.413145:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000a39f3699 697: 1< 0 - - > 2< 2854912 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413147:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000071276d9 00000080:00200000:1.0:1775764495.413148:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 186. 00000080:00200000:1.0:1775764495.413150:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 698 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413163:0:51114:0:(rw26.c:858:ll_write_end()) pos 2859008, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413166:0:51114:0:(rw26.c:916:ll_write_end()) page@0000000016bf03f4[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413169:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @0000000016e56413 17ffffc0001001 3:0 ffff8b23b9f4d000 698 lru 00000080:00200000:1.0:1775764495.413173:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000005bb8b5af 698: 1< 0 - - > 2< 2859008 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413175:0:51114:0:(rw26.c:916:ll_write_end()) end page@0000000016bf03f4 00000080:00200000:1.0:1775764495.413175:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 187. 00000080:00200000:1.0:1775764495.413177:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 699 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413188:0:51114:0:(rw26.c:858:ll_write_end()) pos 2863104, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413191:0:51114:0:(rw26.c:916:ll_write_end()) page@0000000010bec437[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413194:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000002b15eb0e 17ffffc0001001 3:0 ffff8b23b9f4d620 699 lru 00000080:00200000:1.0:1775764495.413199:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@0000000082265b30 699: 1< 0 - - > 2< 2863104 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413201:0:51114:0:(rw26.c:916:ll_write_end()) end page@0000000010bec437 00000080:00200000:1.0:1775764495.413202:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 188. 00000080:00200000:1.0:1775764495.413203:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 700 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413216:0:51114:0:(rw26.c:858:ll_write_end()) pos 2867200, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413219:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000929e4f18[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413221:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000005b10a2d7 17ffffc0001001 3:0 ffff8b23b9f4d9a0 700 lru 00000080:00200000:1.0:1775764495.413225:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000d4f5a887 700: 1< 0 - - > 2< 2867200 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413227:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000929e4f18 00000080:00200000:1.0:1775764495.413228:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 189. 00000080:00200000:1.0:1775764495.413230:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 701 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413252:0:51114:0:(rw26.c:858:ll_write_end()) pos 2871296, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413256:0:51114:0:(rw26.c:916:ll_write_end()) page@0000000063be8493[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413259:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000002160102a 17ffffc0001001 3:0 ffff8b23baeacb60 701 lru 00000080:00200000:1.0:1775764495.413264:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000c6669c62 701: 1< 0 - - > 2< 2871296 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413267:0:51114:0:(rw26.c:916:ll_write_end()) end page@0000000063be8493 00000080:00200000:1.0:1775764495.413268:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 190. 00000080:00200000:1.0:1775764495.413270:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 702 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413283:0:51114:0:(rw26.c:858:ll_write_end()) pos 2875392, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413286:0:51114:0:(rw26.c:916:ll_write_end()) page@0000000057ee45fa[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413289:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @0000000095ed61fc 17ffffc0001001 3:0 ffff8b23baeace00 702 lru 00000080:00200000:1.0:1775764495.413294:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000f22e41c9 702: 1< 0 - - > 2< 2875392 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413295:0:51114:0:(rw26.c:916:ll_write_end()) end page@0000000057ee45fa 00000080:00200000:1.0:1775764495.413296:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 191. 00000080:00200000:1.0:1775764495.413298:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 703 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413316:0:51114:0:(rw26.c:858:ll_write_end()) pos 2879488, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413320:0:51114:0:(rw26.c:916:ll_write_end()) page@000000008b8e9119[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413322:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000003b816b10 17ffffc0001021 2:0 ffff8b23baeacc40 703 lru 00000080:00200000:1.0:1775764495.413326:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000f2b13619 703: 1< 0 - - > 2< 2879488 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413328:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000008b8e9119 00000080:00200000:1.0:1775764495.413329:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 192. 00000080:00200000:1.0:1775764495.413330:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 704 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413348:0:51114:0:(rw26.c:858:ll_write_end()) pos 2883584, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413351:0:51114:0:(rw26.c:916:ll_write_end()) page@0000000056285540[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413354:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000a5f3f8ea 17ffffc0001001 3:0 ffff8b23baeaca80 704 lru 00000080:00200000:1.0:1775764495.413359:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000de2b27c0 704: 1< 0 - - > 2< 2883584 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413361:0:51114:0:(rw26.c:916:ll_write_end()) end page@0000000056285540 00000080:00200000:1.0:1775764495.413362:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 193. 00000080:00200000:1.0:1775764495.413364:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 705 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413377:0:51114:0:(rw26.c:858:ll_write_end()) pos 2887680, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413381:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000787b06b7[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413383:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @0000000032cde8f4 17ffffc0001001 3:0 ffff8b23baeacee0 705 lru 00000080:00200000:1.0:1775764495.413388:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000fb8d8a23 705: 1< 0 - - > 2< 2887680 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413390:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000787b06b7 00000080:00200000:1.0:1775764495.413391:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 194. 00000080:00200000:1.0:1775764495.413392:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 706 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413408:0:51114:0:(rw26.c:858:ll_write_end()) pos 2891776, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413411:0:51114:0:(rw26.c:916:ll_write_end()) page@000000004ce9933d[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413413:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @0000000073ad434f 17ffffc0001001 3:0 ffff8b23baeac1c0 706 lru 00000080:00200000:1.0:1775764495.413418:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000b79807a3 706: 1< 0 - - > 2< 2891776 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413420:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000004ce9933d 00000080:00200000:1.0:1775764495.413421:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 195. 00000080:00200000:1.0:1775764495.413422:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 707 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413432:0:51114:0:(rw26.c:858:ll_write_end()) pos 2895872, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413435:0:51114:0:(rw26.c:916:ll_write_end()) page@000000005541e9de[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413438:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000347a5acd 17ffffc0001001 3:0 ffff8b23baeac540 707 lru 00000080:00200000:1.0:1775764495.413442:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000b2e2b7bb 707: 1< 0 - - > 2< 2895872 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413444:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000005541e9de 00000080:00200000:1.0:1775764495.413445:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 196. 00000080:00200000:1.0:1775764495.413448:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 708 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413492:0:51114:0:(rw26.c:858:ll_write_end()) pos 2899968, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413496:0:51114:0:(rw26.c:916:ll_write_end()) page@000000007a66c721[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413498:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000aa81c5b6 17ffffc0001001 3:0 ffff8b23baeac700 708 lru 00000080:00200000:1.0:1775764495.413502:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000006306cd58 708: 1< 0 - - > 2< 2899968 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413504:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000007a66c721 00000080:00200000:1.0:1775764495.413505:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 197. 00000080:00200000:1.0:1775764495.413507:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 709 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413519:0:51114:0:(rw26.c:858:ll_write_end()) pos 2904064, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413522:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000ac2177c3[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413524:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000008d4d74e2 17ffffc0001001 3:0 ffff8b23baeac7e0 709 lru 00000080:00200000:1.0:1775764495.413530:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@0000000065eca32a 709: 1< 0 - - > 2< 2904064 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413531:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000ac2177c3 00000080:00200000:1.0:1775764495.413533:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 198. 00000080:00200000:1.0:1775764495.413534:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 710 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413544:0:51114:0:(rw26.c:858:ll_write_end()) pos 2908160, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413547:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000cd3c0e47[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413549:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000154314b7 17ffffc0001001 3:0 ffff8b23baeacd20 710 lru 00000080:00200000:1.0:1775764495.413555:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@0000000024007f81 710: 1< 0 - - > 2< 2908160 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413556:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000cd3c0e47 00000080:00200000:1.0:1775764495.413558:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 199. 00000080:00200000:1.0:1775764495.413559:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 711 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413573:0:51114:0:(rw26.c:858:ll_write_end()) pos 2912256, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413576:0:51114:0:(rw26.c:916:ll_write_end()) page@0000000019566041[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413579:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000cecc7b3e 17ffffc0001001 3:0 ffff8b23baeac380 711 lru 00000080:00200000:1.0:1775764495.413584:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000009dd7d869 711: 1< 0 - - > 2< 2912256 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413586:0:51114:0:(rw26.c:916:ll_write_end()) end page@0000000019566041 00000080:00200000:1.0:1775764495.413587:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 200. 00000080:00200000:1.0:1775764495.413589:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 712 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413599:0:51114:0:(rw26.c:858:ll_write_end()) pos 2916352, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413602:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000549cd4e1[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413605:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000bb76b58b 17ffffc0001001 3:0 ffff8b23baeac460 712 lru 00000080:00200000:1.0:1775764495.413610:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000116fe221 712: 1< 0 - - > 2< 2916352 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413612:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000549cd4e1 00000080:00200000:1.0:1775764495.413614:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 201. 00000080:00200000:1.0:1775764495.413615:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 713 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413625:0:51114:0:(rw26.c:858:ll_write_end()) pos 2920448, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413629:0:51114:0:(rw26.c:916:ll_write_end()) page@0000000090b32db9[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413632:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @0000000010e7a31e 17ffffc0001001 3:0 ffff8b23baeac0e0 713 lru 00000080:00200000:1.0:1775764495.413637:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000561e5c93 713: 1< 0 - - > 2< 2920448 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413639:0:51114:0:(rw26.c:916:ll_write_end()) end page@0000000090b32db9 00000080:00200000:1.0:1775764495.413640:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 202. 00000080:00200000:1.0:1775764495.413641:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 714 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413654:0:51114:0:(rw26.c:858:ll_write_end()) pos 2924544, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413657:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000f10fdb4a[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413660:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000009dde222c 17ffffc0001001 3:0 ffff8b23baeac2a0 714 lru 00000080:00200000:1.0:1775764495.413664:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000007eda287c 714: 1< 0 - - > 2< 2924544 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413666:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000f10fdb4a 00000080:00200000:1.0:1775764495.413668:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 203. 00000080:00200000:1.0:1775764495.413669:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 715 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413683:0:51114:0:(rw26.c:858:ll_write_end()) pos 2928640, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413686:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000bb4c2d4d[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413689:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000008a8cde43 17ffffc0001001 3:0 ffff8b23baeac8c0 715 lru 00000080:00200000:1.0:1775764495.413695:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000575c1f03 715: 1< 0 - - > 2< 2928640 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413697:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000bb4c2d4d 00000080:00200000:1.0:1775764495.413698:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 204. 00000080:00200000:1.0:1775764495.413700:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 716 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413710:0:51114:0:(rw26.c:858:ll_write_end()) pos 2932736, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413714:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000b92b8f47[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413717:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000c91e2e9d 17ffffc0001001 3:0 ffff8b23baeac000 716 lru 00000080:00200000:1.0:1775764495.413736:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@0000000068776480 716: 1< 0 - - > 2< 2932736 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413739:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000b92b8f47 00000080:00200000:1.0:1775764495.413740:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 205. 00000080:00200000:1.0:1775764495.413742:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 717 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413757:0:51114:0:(rw26.c:858:ll_write_end()) pos 2936832, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413760:0:51114:0:(rw26.c:916:ll_write_end()) page@000000004da835a9[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413763:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000e980996a 17ffffc0001001 3:0 ffff8b23baeac620 717 lru 00000080:00200000:1.0:1775764495.413769:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000000ad182c9 717: 1< 0 - - > 2< 2936832 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413771:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000004da835a9 00000080:00200000:1.0:1775764495.413772:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 206. 00000080:00200000:1.0:1775764495.413774:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 718 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413789:0:51114:0:(rw26.c:858:ll_write_end()) pos 2940928, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413793:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000c906fe12[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413795:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000747fe1ae 17ffffc0001021 2:0 ffff8b23baeac9a0 718 lru 00000080:00200000:1.0:1775764495.413800:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000982b6650 718: 1< 0 - - > 2< 2940928 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413801:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000c906fe12 00000080:00200000:1.0:1775764495.413803:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 207. 00000080:00200000:1.0:1775764495.413804:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 719 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413825:0:51114:0:(rw26.c:858:ll_write_end()) pos 2945024, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413829:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000b9cbc661[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413831:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000e11d7315 17ffffc0001001 3:0 ffff8b23ac59a8c0 719 lru 00000080:00200000:1.0:1775764495.413836:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000e454a264 719: 1< 0 - - > 2< 2945024 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413838:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000b9cbc661 00000080:00200000:1.0:1775764495.413839:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 208. 00000080:00200000:1.0:1775764495.413841:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 720 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413854:0:51114:0:(rw26.c:858:ll_write_end()) pos 2949120, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413858:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000de5391df[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413860:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000005cc48be4 17ffffc0001001 3:0 ffff8b23ac59a000 720 lru 00000080:00200000:1.0:1775764495.413865:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000210cfd10 720: 1< 0 - - > 2< 2949120 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413867:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000de5391df 00000080:00200000:1.0:1775764495.413868:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 209. 00000080:00200000:1.0:1775764495.413870:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 721 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413882:0:51114:0:(rw26.c:858:ll_write_end()) pos 2953216, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413885:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000c2804eae[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413888:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000f6a4a834 17ffffc0001001 3:0 ffff8b23ac59a620 721 lru 00000080:00200000:1.0:1775764495.413893:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@0000000085207679 721: 1< 0 - - > 2< 2953216 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413895:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000c2804eae 00000080:00200000:1.0:1775764495.413896:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 210. 00000080:00200000:1.0:1775764495.413897:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 722 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413914:0:51114:0:(rw26.c:858:ll_write_end()) pos 2957312, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413917:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000135de745[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413920:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000a4e15db4 17ffffc0001001 3:0 ffff8b23ac59a9a0 722 lru 00000080:00200000:1.0:1775764495.413925:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000009c5f9bfd 722: 1< 0 - - > 2< 2957312 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413927:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000135de745 00000080:00200000:1.0:1775764495.413928:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 211. 00000080:00200000:1.0:1775764495.413930:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 723 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413943:0:51114:0:(rw26.c:858:ll_write_end()) pos 2961408, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413946:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000d2d3562f[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413949:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000db68d896 17ffffc0001001 3:0 ffff8b23ac59ab60 723 lru 00000080:00200000:1.0:1775764495.413953:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000b5b337f2 723: 1< 0 - - > 2< 2961408 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413955:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000d2d3562f 00000080:00200000:1.0:1775764495.413956:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 212. 00000080:00200000:1.0:1775764495.413958:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 724 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.413972:0:51114:0:(rw26.c:858:ll_write_end()) pos 2965504, len 4096, copied 4096 00000080:00200000:1.0:1775764495.413976:0:51114:0:(rw26.c:916:ll_write_end()) page@000000000d90eefe[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.413979:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @0000000097ad41b9 17ffffc0001001 3:0 ffff8b23ac59ae00 724 lru 00000080:00200000:1.0:1775764495.413984:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@0000000097b1444d 724: 1< 0 - - > 2< 2965504 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.413986:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000000d90eefe 00000080:00200000:1.0:1775764495.413988:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 213. 00000080:00200000:1.0:1775764495.413989:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 725 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414005:0:51114:0:(rw26.c:858:ll_write_end()) pos 2969600, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414008:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000ed9139df[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414012:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000ff6053c3 17ffffc0001001 3:0 ffff8b23ac59ac40 725 lru 00000080:00200000:1.0:1775764495.414017:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000004da6e98e 725: 1< 0 - - > 2< 2969600 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414019:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000ed9139df 00000080:00200000:1.0:1775764495.414020:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 214. 00000080:00200000:1.0:1775764495.414022:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 726 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414036:0:51114:0:(rw26.c:858:ll_write_end()) pos 2973696, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414040:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000140929b6[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414042:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000f93c7691 17ffffc0001001 3:0 ffff8b23ac59aa80 726 lru 00000080:00200000:1.0:1775764495.414047:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000009af85486 726: 1< 0 - - > 2< 2973696 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414049:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000140929b6 00000080:00200000:1.0:1775764495.414051:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 215. 00000080:00200000:1.0:1775764495.414053:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 727 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414067:0:51114:0:(rw26.c:858:ll_write_end()) pos 2977792, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414071:0:51114:0:(rw26.c:916:ll_write_end()) page@000000004fe97b2b[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414074:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000007dacdc79 17ffffc0001001 3:0 ffff8b23ac59aee0 727 lru 00000080:00200000:1.0:1775764495.414079:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000001183066c 727: 1< 0 - - > 2< 2977792 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414081:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000004fe97b2b 00000080:00200000:1.0:1775764495.414082:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 216. 00000080:00200000:1.0:1775764495.414084:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 728 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414099:0:51114:0:(rw26.c:858:ll_write_end()) pos 2981888, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414103:0:51114:0:(rw26.c:916:ll_write_end()) page@000000005c787f4e[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414105:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000001d0e6368 17ffffc0001001 3:0 ffff8b23ac59a1c0 728 lru 00000080:00200000:1.0:1775764495.414111:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000009ad674f8 728: 1< 0 - - > 2< 2981888 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414113:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000005c787f4e 00000080:00200000:1.0:1775764495.414114:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 217. 00000080:00200000:1.0:1775764495.414116:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 729 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414132:0:51114:0:(rw26.c:858:ll_write_end()) pos 2985984, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414136:0:51114:0:(rw26.c:916:ll_write_end()) page@000000005bccfee3[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414139:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000042a1bc0 17ffffc0001001 3:0 ffff8b23ac59a540 729 lru 00000080:00200000:1.0:1775764495.414145:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000000e136b32 729: 1< 0 - - > 2< 2985984 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414147:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000005bccfee3 00000080:00200000:1.0:1775764495.414148:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 218. 00000080:00200000:1.0:1775764495.414150:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 730 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414162:0:51114:0:(rw26.c:858:ll_write_end()) pos 2990080, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414165:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000e8f1a4fe[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414168:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000002dbfdf33 17ffffc0001001 3:0 ffff8b23ac59a700 730 lru 00000080:00200000:1.0:1775764495.414173:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000000948a467 730: 1< 0 - - > 2< 2990080 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414176:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000e8f1a4fe 00000080:00200000:1.0:1775764495.414177:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 219. 00000080:00200000:1.0:1775764495.414179:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 731 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414189:0:51114:0:(rw26.c:858:ll_write_end()) pos 2994176, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414193:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000378f8856[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414196:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000004bd5d5a4 17ffffc0001001 3:0 ffff8b23ac59a7e0 731 lru 00000080:00200000:1.0:1775764495.414202:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000c93895b0 731: 1< 0 - - > 2< 2994176 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414204:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000378f8856 00000080:00200000:1.0:1775764495.414205:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 220. 00000080:00200000:1.0:1775764495.414207:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 732 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414218:0:51114:0:(rw26.c:858:ll_write_end()) pos 2998272, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414221:0:51114:0:(rw26.c:916:ll_write_end()) page@0000000087beb570[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414224:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000293d9402 17ffffc0001001 3:0 ffff8b23ac59ad20 732 lru 00000080:00200000:1.0:1775764495.414229:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000d054d124 732: 1< 0 - - > 2< 2998272 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414231:0:51114:0:(rw26.c:916:ll_write_end()) end page@0000000087beb570 00000080:00200000:1.0:1775764495.414232:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 221. 00000080:00200000:1.0:1775764495.414234:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 733 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414248:0:51114:0:(rw26.c:858:ll_write_end()) pos 3002368, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414251:0:51114:0:(rw26.c:916:ll_write_end()) page@000000007050239e[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414255:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000007019ca0d 17ffffc0001021 2:0 ffff8b23ac59a380 733 lru 00000080:00200000:1.0:1775764495.414261:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@0000000011298efc 733: 1< 0 - - > 2< 3002368 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414263:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000007050239e 00000080:00200000:1.0:1775764495.414264:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 222. 00000080:00200000:1.0:1775764495.414266:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 734 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414276:0:51114:0:(rw26.c:858:ll_write_end()) pos 3006464, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414279:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000f64a06cd[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414282:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000f06b1af5 17ffffc0001001 3:0 ffff8b23ac59a460 734 lru 00000080:00200000:1.0:1775764495.414287:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@0000000047bfcd09 734: 1< 0 - - > 2< 3006464 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414289:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000f64a06cd 00000080:00200000:1.0:1775764495.414290:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 223. 00000080:00200000:1.0:1775764495.414292:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 735 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414302:0:51114:0:(rw26.c:858:ll_write_end()) pos 3010560, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414305:0:51114:0:(rw26.c:916:ll_write_end()) page@000000003dc47037[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414308:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @0000000039ab769a 17ffffc0001001 3:0 ffff8b23ac59a0e0 735 lru 00000080:00200000:1.0:1775764495.414313:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000002db5f832 735: 1< 0 - - > 2< 3010560 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414315:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000003dc47037 00000080:00200000:1.0:1775764495.414316:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 224. 00000080:00200000:1.0:1775764495.414318:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 736 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414329:0:51114:0:(rw26.c:858:ll_write_end()) pos 3014656, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414332:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000a1f6cd12[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414335:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000ce02a222 17ffffc0001001 3:0 ffff8b23ac59a2a0 736 lru 00000080:00200000:1.0:1775764495.414341:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000000a44d069 736: 1< 0 - - > 2< 3014656 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414343:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000a1f6cd12 00000080:00200000:1.0:1775764495.414344:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 225. 00000080:00200000:1.0:1775764495.414345:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 737 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414361:0:51114:0:(rw26.c:858:ll_write_end()) pos 3018752, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414365:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000f3625291[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414368:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000961b0d1e 17ffffc0001001 3:0 ffff8b239029d540 737 lru 00000080:00200000:1.0:1775764495.414373:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@0000000075f9525a 737: 1< 0 - - > 2< 3018752 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414375:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000f3625291 00000080:00200000:1.0:1775764495.414376:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 226. 00000080:00200000:1.0:1775764495.414378:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 738 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414389:0:51114:0:(rw26.c:858:ll_write_end()) pos 3022848, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414392:0:51114:0:(rw26.c:916:ll_write_end()) page@0000000033b56b24[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414395:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000a026484a 17ffffc0001001 3:0 ffff8b239029d700 738 lru 00000080:00200000:1.0:1775764495.414400:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000006e43a234 738: 1< 0 - - > 2< 3022848 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414403:0:51114:0:(rw26.c:916:ll_write_end()) end page@0000000033b56b24 00000080:00200000:1.0:1775764495.414404:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 227. 00000080:00200000:1.0:1775764495.414406:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 739 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414416:0:51114:0:(rw26.c:858:ll_write_end()) pos 3026944, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414419:0:51114:0:(rw26.c:916:ll_write_end()) page@000000009e387edc[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414422:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000a602f2bc 17ffffc0001001 3:0 ffff8b239029d7e0 739 lru 00000080:00200000:1.0:1775764495.414428:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@0000000017d953be 739: 1< 0 - - > 2< 3026944 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414430:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000009e387edc 00000080:00200000:1.0:1775764495.414431:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 228. 00000080:00200000:1.0:1775764495.414433:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 740 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414442:0:51114:0:(rw26.c:858:ll_write_end()) pos 3031040, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414446:0:51114:0:(rw26.c:916:ll_write_end()) page@0000000042fc9038[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414449:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000001a14d336 17ffffc0001001 3:0 ffff8b239029dd20 740 lru 00000080:00200000:1.0:1775764495.414488:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000004e450b76 740: 1< 0 - - > 2< 3031040 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414490:0:51114:0:(rw26.c:916:ll_write_end()) end page@0000000042fc9038 00000080:00200000:1.0:1775764495.414492:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 229. 00000080:00200000:1.0:1775764495.414494:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 741 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414506:0:51114:0:(rw26.c:858:ll_write_end()) pos 3035136, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414511:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000d9022c47[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414513:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000009b926fac 17ffffc0001001 3:0 ffff8b239029d380 741 lru 00000080:00200000:1.0:1775764495.414519:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000006988f1ce 741: 1< 0 - - > 2< 3035136 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414522:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000d9022c47 00000080:00200000:1.0:1775764495.414523:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 230. 00000080:00200000:1.0:1775764495.414525:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 742 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414536:0:51114:0:(rw26.c:858:ll_write_end()) pos 3039232, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414540:0:51114:0:(rw26.c:916:ll_write_end()) page@000000005e0a56d2[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414543:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000009938cb8b 17ffffc0001001 3:0 ffff8b239029d460 742 lru 00000080:00200000:1.0:1775764495.414549:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000805b0784 742: 1< 0 - - > 2< 3039232 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414550:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000005e0a56d2 00000080:00200000:1.0:1775764495.414552:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 231. 00000080:00200000:1.0:1775764495.414553:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 743 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414565:0:51114:0:(rw26.c:858:ll_write_end()) pos 3043328, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414569:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000476ba711[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414571:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000006c6d9eff 17ffffc0001001 3:0 ffff8b239029d0e0 743 lru 00000080:00200000:1.0:1775764495.414577:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000008f26986e 743: 1< 0 - - > 2< 3043328 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414579:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000476ba711 00000080:00200000:1.0:1775764495.414580:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 232. 00000080:00200000:1.0:1775764495.414581:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 744 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414592:0:51114:0:(rw26.c:858:ll_write_end()) pos 3047424, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414595:0:51114:0:(rw26.c:916:ll_write_end()) page@000000005c78f48b[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414599:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000001e0f389b 17ffffc0001001 3:0 ffff8b239029d2a0 744 lru 00000080:00200000:1.0:1775764495.414605:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000006a02a2a1 744: 1< 0 - - > 2< 3047424 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414607:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000005c78f48b 00000080:00200000:1.0:1775764495.414608:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 233. 00000080:00200000:1.0:1775764495.414609:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 745 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414620:0:51114:0:(rw26.c:858:ll_write_end()) pos 3051520, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414624:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000d612ab89[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414626:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @0000000044c32492 17ffffc0001001 3:0 ffff8b239029d8c0 745 lru 00000080:00200000:1.0:1775764495.414632:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000b2ec9e91 745: 1< 0 - - > 2< 3051520 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414634:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000d612ab89 00000080:00200000:1.0:1775764495.414635:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 234. 00000080:00200000:1.0:1775764495.414636:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 746 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414646:0:51114:0:(rw26.c:858:ll_write_end()) pos 3055616, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414650:0:51114:0:(rw26.c:916:ll_write_end()) page@0000000086dc292a[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414652:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @0000000058171b08 17ffffc0001001 3:0 ffff8b239029d000 746 lru 00000080:00200000:1.0:1775764495.414658:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000fb54c337 746: 1< 0 - - > 2< 3055616 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414660:0:51114:0:(rw26.c:916:ll_write_end()) end page@0000000086dc292a 00000080:00200000:1.0:1775764495.414661:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 235. 00000080:00200000:1.0:1775764495.414663:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 747 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414674:0:51114:0:(rw26.c:858:ll_write_end()) pos 3059712, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414677:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000b93f8f5b[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414680:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @0000000045141293 17ffffc0001001 3:0 ffff8b239029d620 747 lru 00000080:00200000:1.0:1775764495.414686:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000bbb806f0 747: 1< 0 - - > 2< 3059712 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414688:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000b93f8f5b 00000080:00200000:1.0:1775764495.414689:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 236. 00000080:00200000:1.0:1775764495.414690:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 748 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.414704:0:51114:0:(rw26.c:858:ll_write_end()) pos 3063808, len 4096, copied 4096 00000080:00200000:1.0:1775764495.414707:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000b0203459[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.414710:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000003699b7e5 17ffffc0001021 2:0 ffff8b239029d9a0 748 lru 00000080:00200000:1.0:1775764495.414715:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000a9f03624 748: 1< 0 - - > 2< 3063808 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.414717:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000b0203459 00000080:00200000:1.0:1775764495.414718:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 237. 00000080:00200000:1.0:1775764495.414720:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 749 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.428333:0:51114:0:(rw26.c:858:ll_write_end()) pos 3067904, len 4096, copied 4096 00000080:00200000:1.0:1775764495.428347:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000c61a550f[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.428352:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000004b63b2fd 17ffffc0001001 3:0 ffff8b239029db60 749 lru 00000080:00200000:1.0:1775764495.428359:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000004d073e7d 749: 1< 0 - - > 2< 3067904 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.428362:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000c61a550f 00000080:00200000:1.0:1775764495.428363:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 238. 00000080:00200000:1.0:1775764495.428367:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 750 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.428393:0:51114:0:(rw26.c:858:ll_write_end()) pos 3072000, len 4096, copied 4096 00000080:00200000:1.0:1775764495.428397:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000faa51ba1[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.428400:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000a7cccb87 17ffffc0001001 3:0 ffff8b239029de00 750 lru 00000080:00200000:1.0:1775764495.428405:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000af38ac38 750: 1< 0 - - > 2< 3072000 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.428407:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000faa51ba1 00000080:00200000:1.0:1775764495.428409:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 239. 00000080:00200000:1.0:1775764495.428410:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 751 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.428425:0:51114:0:(rw26.c:858:ll_write_end()) pos 3076096, len 4096, copied 4096 00000080:00200000:1.0:1775764495.428429:0:51114:0:(rw26.c:916:ll_write_end()) page@000000005014cdb7[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.428432:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000ad708678 17ffffc0001001 3:0 ffff8b239029dc40 751 lru 00000080:00200000:1.0:1775764495.428436:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000ee0e10a5 751: 1< 0 - - > 2< 3076096 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.428439:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000005014cdb7 00000080:00200000:1.0:1775764495.428440:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 240. 00000080:00200000:1.0:1775764495.428442:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 752 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.428479:0:51114:0:(rw26.c:858:ll_write_end()) pos 3080192, len 4096, copied 4096 00000080:00200000:1.0:1775764495.428484:0:51114:0:(rw26.c:916:ll_write_end()) page@000000009fdfaa2e[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.428487:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000dc57e274 17ffffc0001001 3:0 ffff8b239029da80 752 lru 00000080:00200000:1.0:1775764495.428492:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000c3363ecd 752: 1< 0 - - > 2< 3080192 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.428494:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000009fdfaa2e 00000080:00200000:1.0:1775764495.428495:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 241. 00000080:00200000:1.0:1775764495.428497:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 753 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.428512:0:51114:0:(rw26.c:858:ll_write_end()) pos 3084288, len 4096, copied 4096 00000080:00200000:1.0:1775764495.428516:0:51114:0:(rw26.c:916:ll_write_end()) page@000000008d7bb775[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.428518:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000009bb8d32e 17ffffc0001001 3:0 ffff8b239029dee0 753 lru 00000080:00200000:1.0:1775764495.428523:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000004ac6ecd9 753: 1< 0 - - > 2< 3084288 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.428525:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000008d7bb775 00000080:00200000:1.0:1775764495.428526:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 242. 00000080:00200000:1.0:1775764495.428528:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 754 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.428541:0:51114:0:(rw26.c:858:ll_write_end()) pos 3088384, len 4096, copied 4096 00000080:00200000:1.0:1775764495.428545:0:51114:0:(rw26.c:916:ll_write_end()) page@000000000e473f79[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.428548:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000006be83e1a 17ffffc0001001 3:0 ffff8b239029d1c0 754 lru 00000080:00200000:1.0:1775764495.428553:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@0000000018119db0 754: 1< 0 - - > 2< 3088384 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.428555:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000000e473f79 00000080:00200000:1.0:1775764495.428556:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 243. 00000080:00200000:1.0:1775764495.428557:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 755 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.428585:0:51114:0:(rw26.c:858:ll_write_end()) pos 3092480, len 4096, copied 4096 00000080:00200000:1.0:1775764495.428589:0:51114:0:(rw26.c:916:ll_write_end()) page@0000000021fb87cb[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.428591:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000666a5dc8 17ffffc0001001 3:0 ffff8b23ac4dc7e0 755 lru 00000080:00200000:1.0:1775764495.428596:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000b5315c06 755: 1< 0 - - > 2< 3092480 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.428598:0:51114:0:(rw26.c:916:ll_write_end()) end page@0000000021fb87cb 00000080:00200000:1.0:1775764495.428599:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 244. 00000080:00200000:1.0:1775764495.428601:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 756 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.428627:0:51114:0:(rw26.c:858:ll_write_end()) pos 3096576, len 4096, copied 4096 00000080:00200000:1.0:1775764495.428632:0:51114:0:(rw26.c:916:ll_write_end()) page@0000000078478c19[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.428635:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000000e77849c 17ffffc0001001 3:0 ffff8b23ac4dcd20 756 lru 00000080:00200000:1.0:1775764495.428640:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000de4b094a 756: 1< 0 - - > 2< 3096576 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.428642:0:51114:0:(rw26.c:916:ll_write_end()) end page@0000000078478c19 00000080:00200000:1.0:1775764495.428643:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 245. 00000080:00200000:1.0:1775764495.428645:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 757 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.428659:0:51114:0:(rw26.c:858:ll_write_end()) pos 3100672, len 4096, copied 4096 00000080:00200000:1.0:1775764495.428663:0:51114:0:(rw26.c:916:ll_write_end()) page@000000009ca5d217[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.428666:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000706816ed 17ffffc0001001 3:0 ffff8b23ac4dc380 757 lru 00000080:00200000:1.0:1775764495.428670:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@0000000034ec7929 757: 1< 0 - - > 2< 3100672 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.428672:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000009ca5d217 00000080:00200000:1.0:1775764495.428673:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 246. 00000080:00200000:1.0:1775764495.428675:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 758 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.428689:0:51114:0:(rw26.c:858:ll_write_end()) pos 3104768, len 4096, copied 4096 00000080:00200000:1.0:1775764495.428694:0:51114:0:(rw26.c:916:ll_write_end()) page@0000000010555211[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.428696:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000000d1ef6f5 17ffffc0001001 3:0 ffff8b23ac4dc460 758 lru 00000080:00200000:1.0:1775764495.428701:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000718e4d63 758: 1< 0 - - > 2< 3104768 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.428703:0:51114:0:(rw26.c:916:ll_write_end()) end page@0000000010555211 00000080:00200000:1.0:1775764495.428705:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 247. 00000080:00200000:1.0:1775764495.428707:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 759 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.428717:0:51114:0:(rw26.c:858:ll_write_end()) pos 3108864, len 4096, copied 4096 00000080:00200000:1.0:1775764495.428720:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000bd573ef0[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.428734:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000b372c885 17ffffc0001001 3:0 ffff8b23ac4dc0e0 759 lru 00000080:00200000:1.0:1775764495.428740:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000f1a8491b 759: 1< 0 - - > 2< 3108864 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.428742:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000bd573ef0 00000080:00200000:1.0:1775764495.428743:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 248. 00000080:00200000:1.0:1775764495.428745:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 760 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.428758:0:51114:0:(rw26.c:858:ll_write_end()) pos 3112960, len 4096, copied 4096 00000080:00200000:1.0:1775764495.428762:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000a00c33e8[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.428765:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000bdee63f8 17ffffc0001001 3:0 ffff8b23ac4dc2a0 760 lru 00000080:00200000:1.0:1775764495.429076:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000c5c86c54 760: 1< 0 - - > 2< 3112960 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.429080:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000a00c33e8 00000080:00200000:1.0:1775764495.429082:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 249. 00000080:00200000:1.0:1775764495.429085:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 761 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.429101:0:51114:0:(rw26.c:858:ll_write_end()) pos 3117056, len 4096, copied 4096 00000080:00200000:1.0:1775764495.429105:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000fa5a2dfe[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.429108:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000afd52fe5 17ffffc0001001 3:0 ffff8b23ac4dc8c0 761 lru 00000080:00200000:1.0:1775764495.429113:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000006a165795 761: 1< 0 - - > 2< 3117056 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.429115:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000fa5a2dfe 00000080:00200000:1.0:1775764495.429116:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 250. 00000080:00200000:1.0:1775764495.429118:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 762 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.429130:0:51114:0:(rw26.c:858:ll_write_end()) pos 3121152, len 4096, copied 4096 00000080:00200000:1.0:1775764495.429133:0:51114:0:(rw26.c:916:ll_write_end()) page@0000000082b724bf[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.429136:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000ecac7534 17ffffc0001001 3:0 ffff8b23ac4dc000 762 lru 00000080:00200000:1.0:1775764495.429141:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000f5d1fa27 762: 1< 0 - - > 2< 3121152 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.429143:0:51114:0:(rw26.c:916:ll_write_end()) end page@0000000082b724bf 00000080:00200000:1.0:1775764495.429144:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 251. 00000080:00200000:1.0:1775764495.429146:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 763 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.429166:0:51114:0:(rw26.c:858:ll_write_end()) pos 3125248, len 4096, copied 4096 00000080:00200000:1.0:1775764495.429170:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000aea50ba4[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.429173:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @000000001db24e88 17ffffc0001021 2:0 ffff8b23ac4dc620 763 lru 00000080:00200000:1.0:1775764495.429179:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000616df952 763: 1< 0 - - > 2< 3125248 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.429181:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000aea50ba4 00000080:00200000:1.0:1775764495.429182:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 252. 00000080:00200000:1.0:1775764495.429183:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 764 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.429197:0:51114:0:(rw26.c:858:ll_write_end()) pos 3129344, len 4096, copied 4096 00000080:00200000:1.0:1775764495.429201:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000f52db707[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.429203:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000027cab07 17ffffc0001001 3:0 ffff8b23ac4dc9a0 764 lru 00000080:00200000:1.0:1775764495.429208:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@0000000027d7ad6a 764: 1< 0 - - > 2< 3129344 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.429209:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000f52db707 00000080:00200000:1.0:1775764495.429211:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 253. 00000080:00200000:1.0:1775764495.429213:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 765 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.429225:0:51114:0:(rw26.c:858:ll_write_end()) pos 3133440, len 4096, copied 4096 00000080:00200000:1.0:1775764495.429228:0:51114:0:(rw26.c:916:ll_write_end()) page@00000000b3375bdb[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.429230:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @0000000052176450 17ffffc0001001 3:0 ffff8b23ac4dcb60 765 lru 00000080:00200000:1.0:1775764495.429234:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000e6173311 765: 1< 0 - - > 2< 3133440 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.429236:0:51114:0:(rw26.c:916:ll_write_end()) end page@00000000b3375bdb 00000080:00200000:1.0:1775764495.429237:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 254. 00000080:00200000:1.0:1775764495.429238:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 766 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.429252:0:51114:0:(rw26.c:858:ll_write_end()) pos 3137536, len 4096, copied 4096 00000080:00200000:1.0:1775764495.429257:0:51114:0:(rw26.c:916:ll_write_end()) page@0000000006674752[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.429259:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000673ccdec 17ffffc0001001 3:0 ffff8b23ac4dce00 766 lru 00000080:00200000:1.0:1775764495.429264:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@000000007ec22ad2 766: 1< 0 - - > 2< 3137536 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.429266:0:51114:0:(rw26.c:916:ll_write_end()) end page@0000000006674752 00000080:00200000:1.0:1775764495.429266:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 255. 00000080:00200000:1.0:1775764495.429269:0:51114:0:(rw26.c:663:ll_write_begin()) Writing 767 of 0 to 4096 bytes 00000080:00200000:1.0:1775764495.429288:0:51114:0:(rw26.c:858:ll_write_end()) pos 3141632, len 4096, copied 4096 00000080:00200000:1.0:1775764495.429292:0:51114:0:(rw26.c:916:ll_write_end()) page@000000000d39148c[3 00000000bc534084 2 1 00000000917d59e6] 00000080:00200000:1.0:1775764495.429294:0:51114:0:(rw26.c:916:ll_write_end()) vmpage @00000000d50eb0a8 17ffffc0001001 3:0 ffff8b23ac4dcc40 767 lru 00000080:00200000:1.0:1775764495.429299:0:51114:0:(rw26.c:916:ll_write_end()) osc-page@00000000d93ea566 767: 1< 0 - - > 2< 3141632 0 0 0x0 0x0 | 000000004bfef949 000000003cb640e4 > 3< 0 0 > 4< 0 1 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764495.429300:0:51114:0:(rw26.c:916:ll_write_end()) end page@000000000d39148c 00000080:00200000:1.0:1775764495.429301:0:51114:0:(rw26.c:916:ll_write_end()) queued cl_page: 256. 00000080:00200000:1.0:1775764495.429306:0:51114:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 256, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764495.429425:0:51114:0:(osc_cache.c:848:osc_extent_find()) ### extent: 00000000d92ed129 ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000005a7b8af4/0x73cfd86cfc17b0db lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f2a4 expref: -99 pid: 51114 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764495.429519:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 15, dirtied 15 00000080:00200000:1.0:1775764495.429536:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 15, dirtied 15 00000080:00200000:1.0:1775764495.429552:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 15, dirtied 15 00000080:00200000:1.0:1775764495.429569:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 15, dirtied 15 00000080:00200000:1.0:1775764495.429586:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 15, dirtied 15 00000080:00200000:1.0:1775764495.429602:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 15, dirtied 15 00000080:00200000:1.0:1775764495.430399:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 15, dirtied 15 00000080:00200000:1.0:1775764495.430422:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 15, dirtied 15 00000080:00200000:1.0:1775764495.430439:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 15, dirtied 15 00000080:00200000:1.0:1775764495.430492:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 15, dirtied 15 00000080:00200000:1.0:1775764495.430512:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 15, dirtied 15 00000080:00200000:1.0:1775764495.430529:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 15, dirtied 15 00000080:00200000:1.0:1775764495.430545:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 15, dirtied 15 00000080:00200000:1.0:1775764495.430561:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 15, dirtied 15 00000080:00200000:1.0:1775764495.430579:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 15, dirtied 15 00000080:00200000:1.0:1775764495.430596:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 15, dirtied 15 00000080:00200000:1.0:1775764495.430898:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 15, dirtied 15 00000080:00200000:1.0:1775764495.430909:0:51114:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a65d38, count 1, dirtied 1 00000080:00200000:1.0:1775764495.430914:0:51114:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 256 pages 1048576 bytes, tot: 1048576 00000080:00200000:1.0:1775764495.430918:0:51114:0:(vvp_io.c:1353:vvp_io_write_start()) f18.sanity-pcc: write bytes 0, result: 1048576 00000020:00010000:1.0:1775764495.430951:0:51114:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb7b78 (ffff8b2387cbab48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764495.430954:0:51114:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb7b78 (ffff8b2387cbab48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764495.430958:0:51114:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a3188b0 (ffff8b2387cbaea8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764495.430967:0:51114:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000005a7b8af4/0x73cfd86cfc17b0db lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304f2a4 expref: -99 pid: 51114 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764495.430983:0:51114:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000005a7b8af4/0x73cfd86cfc17b0db lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f2a4 expref: -99 pid: 51114 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000020:00010000:1.0:1775764495.431001:0:51114:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb7b78 (ffff8b2387cbab48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764495.431004:0:51114:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a3188b0 (ffff8b2387cbab48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764495.431015:0:51114:0:(vvp_io.c:517:vvp_io_advance()) advancing 1048576 bytes 00000080:00200000:1.0:1775764495.431022:0:51114:0:(file.c:2173:ll_file_io_generic()) Range unlock [512, 767] 00000080:00200000:1.0:1775764495.431056:0:51114:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x1d:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764495.431061:0:51114:0:(file.c:2206:ll_file_io_generic()) f18.sanity-pcc: 2 io complete with rc: 0, result: 1048576, restart: 0 00000008:00010000:1.0:1775764495.431201:0:43507:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 00000000d92ed129 ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000005a7b8af4/0x73cfd86cfc17b0db lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f2a4 expref: -99 pid: 51114 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000008:00010000:1.0:1775764495.431215:0:43507:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000d92ed129 ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000005a7b8af4/0x73cfd86cfc17b0db lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f2a4 expref: -99 pid: 51114 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1775764495.433869:0:43507:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (2097152 2101247) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000005a7b8af4/0x73cfd86cfc17b0db lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f2a4 expref: -99 pid: 51114 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000008:00100000:1.0:1775764495.436937:0:43507:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23864fce00 - object 0x280000400:264 offset 2097152<>3145728 00000100:00100000:1.0:1775764495.438592:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098361216:192.168.202.128@tcp:4:dd.0 00000008:00010000:1.0:1775764495.509796:0:2377:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 00000000d92ed129 ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000005a7b8af4/0x73cfd86cfc17b0db lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f2a4 expref: -99 pid: 51114 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000008:00010000:1.0:1775764495.510031:0:2377:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000d92ed129 ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000005a7b8af4/0x73cfd86cfc17b0db lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f2a4 expref: -99 pid: 51114 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000100:00100000:1.0:1775764495.510173:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098361216:192.168.202.128@tcp:4:dd.0 00000100:00100000:1.0:1775764495.604030:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764495.611258:0:51119:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:51119:1862022098361984:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764495.611279:0:51119:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CW ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7afae00/0x73cfd86cfc17b0e9 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f2c0 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.611301:0:51119:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7afae00/0x73cfd86cfc17b0e9 lrc: 4/1,0 mode: --/CW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f2c0 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.611317:0:51119:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7afae00/0x73cfd86cfc17b0e9 lrc: 4/1,0 mode: --/CW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f2c0 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.611330:0:51119:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7afae00/0x73cfd86cfc17b0e9 lrc: 4/1,0 mode: CW/CW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f2c0 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.611340:0:51119:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7afae00/0x73cfd86cfc17b0e9 lrc: 4/1,0 mode: CW/CW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f2c0 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.611354:0:51119:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7afae00/0x73cfd86cfc17b0e9 lrc: 5/1,1 mode: CW/CW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f2c0 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.611362:0:51119:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7afae00/0x73cfd86cfc17b0e9 lrc: 5/1,1 mode: CW/CW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f2c0 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.611380:0:51119:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7afae00/0x73cfd86cfc17b0e9 lrc: 4/0,1 mode: CW/CW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f2c0 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764495.611392:0:51119:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b2382dac700 x1862022098361984/t34359738641(34359738641) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/664 e 0 to 0 dl 1775764511 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764495.611406:0:51119:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382dac700 x1862022098361984/t34359738641(34359738641) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/664 e 0 to 0 dl 1775764511 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764495.611423:0:51119:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7afae00/0x73cfd86cfc17b0e9 lrc: 3/0,1 mode: CW/CW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f2c0 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.611432:0:51119:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 8 res 8589949617/29 (0 0) 00000080:00200000:1.0:1775764495.611440:0:51119:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1775764495.611551:0:51119:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x1d:0x0](ffff8b2389a65b88) for lock 0x73cfd86cfc17b0e9 00000080:00010000:1.0:1775764495.611558:0:51119:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b0e9 from it ffffa60b01e7bbf0 00010000:00010000:1.0:1775764495.611561:0:51119:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7afae00/0x73cfd86cfc17b0e9 lrc: 3/0,1 mode: CW/CW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f2c0 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.611571:0:51119:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7afae00/0x73cfd86cfc17b0e9 lrc: 2/0,0 mode: CW/CW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f2c0 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764495.611593:0:51119:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382dac700 x1862022098361984/t34359738641(34359738641) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764511 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764495.611844:0:51119:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab1:0x1d:0x0](ffff8b2389a65b88) cmd=c00c66f3 arg=7fffed495674 00000002:00010000:1.0:1775764495.611866:0:51119:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x1d:0x0]) in obj [0x200003ab1:0x1d:0x0], intent: open flags 01004040000002 00010000:00010000:1.0:1775764495.611879:0:51119:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7afae00/0x73cfd86cfc17b0e9 lrc: 2/0,0 mode: CW/CW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0304f2c0 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764495.611940:0:51119:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7afae00/0x73cfd86cfc17b0e9 lrc: 1/0,0 mode: --/CW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304f2c0 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764495.611946:0:51119:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764495.611949:0:51119:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7afae00/0x73cfd86cfc17b0e9 lrc: 0/0,0 mode: --/CW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304f2c0 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764495.611978:0:51119:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7a45200/0x73cfd86cfc17b0f0 lrc: 3/1,0 mode: --/CR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 51119 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.611984:0:51119:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x81000000001000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7a45200/0x73cfd86cfc17b0f0 lrc: 3/1,0 mode: --/CR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 51119 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764495.612000:0:51119:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764495.612006:0:51119:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fea00 x1862022098362368/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/131984 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764495.612016:0:51119:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7a45200/0x73cfd86cfc17b0f0 lrc: 3/1,0 mode: --/CR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x81000000000000 pid: 51119 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764495.612039:0:51119:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:51119:1862022098362368:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764495.612286:0:51119:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6f6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764495.616091:0:51119:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:51119:1862022098362368:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764495.616128:0:51119:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PW ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7a45200/0x73cfd86cfc17b0f0 lrc: 4/1,0 mode: --/CR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e0304f2d5 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.616144:0:51119:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7a45200/0x73cfd86cfc17b0f0 lrc: 4/1,0 mode: --/PW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e0304f2d5 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.616158:0:51119:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7a45200/0x73cfd86cfc17b0f0 lrc: 4/1,0 mode: PW/PW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e0304f2d5 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.616171:0:51119:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7a45200/0x73cfd86cfc17b0f0 lrc: 4/1,0 mode: PW/PW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e0304f2d5 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.616189:0:51119:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7a45200/0x73cfd86cfc17b0f0 lrc: 5/1,1 mode: PW/PW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e0304f2d5 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.616198:0:51119:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7a45200/0x73cfd86cfc17b0f0 lrc: 5/1,1 mode: PW/PW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e0304f2d5 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.616211:0:51119:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7a45200/0x73cfd86cfc17b0f0 lrc: 4/0,1 mode: PW/PW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e0304f2d5 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764495.616223:0:51119:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=600002b, status=0 req@ffff8b23864fea00 x1862022098362368/t34359738642(34359738642) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/664 e 0 to 0 dl 1775764511 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764495.616236:0:51119:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fea00 x1862022098362368/t34359738642(34359738642) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/664 e 0 to 0 dl 1775764511 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764495.616252:0:51119:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7a45200/0x73cfd86cfc17b0f0 lrc: 3/0,1 mode: PW/PW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e0304f2d5 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.616264:0:51119:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 2 res 8589949617/29 (0 0) 00000080:00200000:1.0:1775764495.616271:0:51119:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab1:0x1d:0x0] suppgids 0 0: rc 0 00000002:00100000:1.0:1775764495.616290:0:51119:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fea00 x1862022098362368/t34359738642(34359738642) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764511 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1775764495.616298:0:51119:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x1d:0x0](ffff8b2389a65b88) for lock 0x73cfd86cfc17b0f0 00000080:00010000:1.0:1775764495.616305:0:51119:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b0f0 from it ffffa60b01e7bcc0 00010000:00010000:1.0:1775764495.616308:0:51119:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7a45200/0x73cfd86cfc17b0f0 lrc: 3/0,1 mode: PW/PW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e0304f2d5 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.616315:0:51119:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7a45200/0x73cfd86cfc17b0f0 lrc: 2/0,0 mode: PW/PW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e0304f2d5 expref: -99 pid: 51119 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200080:1.0:1775764495.617052:0:51119:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab1:0x1d:0x0](ffff8b2389a65b88) cmd=c00c66f3 arg=231a2a0 00000080:00200008:1.0:1775764495.618173:0:51119:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 0, count: 1048576 00000080:00200000:1.0:1775764495.618225:0:51119:0:(file.c:2058:ll_file_io_generic()) f18.sanity-pcc: read ppos: 0, bytes: 1048576 00000080:00200000:1.0:1775764495.618231:0:51119:0:(file.c:1815:ll_io_set_mirror()) f18.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764495.618241:0:51119:0:(cl_io.c:214:cl_io_rw_init()) header@00000000069c61f7[0x110, 1025, [0x200003ab1:0x1d:0x0] hash] 00000020:00200000:1.0:1775764495.618243:0:51119:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 1048576) 2048 0 00000080:00200000:1.0:1775764495.618249:0:51119:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x1d:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00010000:00010000:1.0:1775764495.618273:0:51119:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 31 res 8589949617/29 (0 0) 00000080:00010000:1.0:1775764495.618284:0:51119:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab1:0x1d:0x0](ffff8b2389a65b88) 00000002:00010000:1.0:1775764495.618294:0:51119:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x1d:0x0]) in obj [0x200003ab1:0x1d:0x0], intent: layout flags 0 00010000:00010000:1.0:1775764495.618356:0:51119:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7a45800/0x73cfd86cfc17b0f7 lrc: 3/1,0 mode: --/CR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 51119 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.618365:0:51119:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7a45800/0x73cfd86cfc17b0f7 lrc: 3/1,0 mode: --/CR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 51119 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764495.618554:0:51119:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7a45800/0x73cfd86cfc17b0f7 lrc: 3/1,0 mode: --/CR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 51119 initiator: MDT0 lru_score: 0 lru_type: 0 00000080:00200008:1.0:1775764495.649057:0:51119:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 3145728, count: 1048576 00000100:00100000:1.0:1775764495.664642:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098362624:192.168.202.128@tcp:1:lfs.0 00000100:00100000:1.0:1775764495.668980:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118973696 00000100:00100000:1.0:1775764495.668992:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118973696 00000100:00100000:1.0:1775764495.669040:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 75 00000100:00100000:1.0:1775764495.669051:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daf800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42945:x1862022118973696:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764495.669078:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000005a7b8af4/0x73cfd86cfc17b0db lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e0304f2a4 expref: -99 pid: 51114 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1775764495.669171:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764495.669177:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daf800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42945:x1862022118973696:12345-192.168.202.128@tcp:104: Request processed in 129us (1257us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764495.669188:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 75 00000100:00100000:1.0:1775764495.682265:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098362624:192.168.202.128@tcp:1:lfs.0 00000100:00100000:1.0:1775764495.694252:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118973824 00000100:00100000:1.0:1775764495.694265:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118973824 00000100:00100000:1.0:1775764495.694322:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 76 00000100:00100000:1.0:1775764495.694332:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dae300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42328:x1862022118973824:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764495.694351:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7a45800/0x73cfd86cfc17b0f7 lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0304f2e3 expref: -99 pid: 51119 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764495.694444:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764495.694502:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dae300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42328:x1862022118973824:12345-192.168.202.128@tcp:104: Request processed in 120us (448us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764495.694512:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 76 00010000:00010000:1.0:1775764496.199435:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764496.199467:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9ceba00/0x73cfd86cfc17b10c lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304f353 expref: -99 pid: 51142 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764496.199482:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9ceba00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764496.199487:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9ceba00/0x73cfd86cfc17b10c lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304f353 expref: -99 pid: 51142 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764496.199495:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9ceba00/0x73cfd86cfc17b10c lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304f353 expref: -99 pid: 51142 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764496.199505:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9ceba00/0x73cfd86cfc17b10c lrc: 3/0,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0304f353 expref: -99 pid: 51142 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764496.199539:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9cea800/0x73cfd86cfc17b105 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304f33e expref: -99 pid: 51142 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764496.199587:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9ceba00/0x73cfd86cfc17b10c lrc: 2/0,0 mode: --/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304f353 expref: -99 pid: 51142 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764496.199595:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764496.199621:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9ceba00/0x73cfd86cfc17b10c lrc: 1/0,0 mode: --/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304f353 expref: -99 pid: 51142 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764496.199629:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9ceba00/0x73cfd86cfc17b10c lrc: 0/0,0 mode: --/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304f353 expref: -99 pid: 51142 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764496.199652:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764496.199653:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764496.199655:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764496.199682:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098364544:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764496.349663:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118974464 00000100:00100000:1.0:1775764496.349676:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118974464 00000100:00100000:1.0:1775764496.349743:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 77 00000100:00100000:1.0:1775764496.349755:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dac380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42326:x1862022118974464:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764496.349778:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9cea800/0x73cfd86cfc17b105 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0304f33e expref: -99 pid: 51142 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764496.349863:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764496.349870:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dac380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42326:x1862022118974464:12345-192.168.202.128@tcp:104: Request processed in 118us (473us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764496.349879:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 77 00000100:00100000:1.0:1775764496.371843:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118974592 00000100:00100000:1.0:1775764496.371852:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118974592 00000100:00100000:1.0:1775764496.371943:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 78 00000100:00100000:1.0:1775764496.372224:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dac700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42324:x1862022118974592:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764496.372247:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23a7afb800/0x73cfd86cfc17b0fe lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0304f2f8 expref: -99 pid: 51119 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764496.372387:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764496.372394:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dac700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42324:x1862022118974592:12345-192.168.202.128@tcp:104: Request processed in 442us (661us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764496.372401:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 78 00000100:00100000:1.0:1775764496.657538:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764496.880080:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764496.888186:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118975488 00000100:00100000:1.0:1775764496.888196:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118975488 00000100:00100000:1.0:1775764496.888255:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 81 00000100:00100000:1.0:1775764496.888265:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dae300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:44302:x1862022118975488:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764496.888282:0:46140:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ca4d19b/0x73cfd86cfc17b160 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f441 expref: -99 pid: 51151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764496.888297:0:46140:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ca4d19b/0x73cfd86cfc17b160 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f441 expref: -99 pid: 51151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764496.888307:0:46140:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ca4d19b/0x73cfd86cfc17b160 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f441 expref: -99 pid: 51151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764496.888324:0:46140:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ca4d19b/0x73cfd86cfc17b160 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e0304f441 expref: -99 pid: 51151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764496.888334:0:46140:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ca4d19b/0x73cfd86cfc17b160 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e0304f441 expref: -99 pid: 51151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764496.888344:0:46140:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ca4d19b/0x73cfd86cfc17b160 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x80000000000 nid: local remote: 0x57b8ff0e0304f441 expref: -99 pid: 51151 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764496.888363:0:46140:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b23b9ceba00) 00000100:00100000:1.0:1775764496.888423:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dae300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:44302:x1862022118975488:12345-192.168.202.128@tcp:105: Request processed in 159us (348us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764496.888431:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 81 00000080:00200000:1.0:1775764496.925566:0:51151:0:(rw26.c:858:ll_write_end()) pos 1093632, len 4096, copied 4096 00000080:00200000:1.0:1775764496.925581:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000d562bf92[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.925585:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000cecac960 17ffffc0001021 2:0 ffff8b23ac4dca80 267 lru 00000080:00200000:1.0:1775764496.925592:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000a4c093f5 267: 1< 0 - - > 2< 1093632 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.925892:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000d562bf92 00000080:00200000:1.0:1775764496.925896:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 12. 00000080:00200000:1.0:1775764496.925903:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 268 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.925939:0:51151:0:(rw26.c:858:ll_write_end()) pos 1097728, len 4096, copied 4096 00000080:00200000:1.0:1775764496.925944:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000029fbb592[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.925947:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000003006db22 17ffffc0001001 3:0 ffff8b23ac4dcee0 268 lru 00000080:00200000:1.0:1775764496.925952:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000a7629a8e 268: 1< 0 - - > 2< 1097728 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.925955:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000029fbb592 00000080:00200000:1.0:1775764496.925956:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 13. 00000080:00200000:1.0:1775764496.925958:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 269 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.925974:0:51151:0:(rw26.c:858:ll_write_end()) pos 1101824, len 4096, copied 4096 00000080:00200000:1.0:1775764496.925978:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000026d79f7b[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.925980:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000982ee95f 17ffffc0001001 3:0 ffff8b23ac4dc1c0 269 lru 00000080:00200000:1.0:1775764496.925985:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000003a4d578e 269: 1< 0 - - > 2< 1101824 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.925988:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000026d79f7b 00000080:00200000:1.0:1775764496.925989:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 14. 00000080:00200000:1.0:1775764496.925990:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 270 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.926005:0:51151:0:(rw26.c:858:ll_write_end()) pos 1105920, len 4096, copied 4096 00000080:00200000:1.0:1775764496.926009:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000d08531ba[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.926012:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000022b4a0c0 17ffffc0001001 3:0 ffff8b23ac4dc540 270 lru 00000080:00200000:1.0:1775764496.926017:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000004ec46b37 270: 1< 0 - - > 2< 1105920 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.926019:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000d08531ba 00000080:00200000:1.0:1775764496.926020:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 15. 00000080:00200000:1.0:1775764496.926022:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 271 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.926037:0:51151:0:(rw26.c:858:ll_write_end()) pos 1110016, len 4096, copied 4096 00000080:00200000:1.0:1775764496.926041:0:51151:0:(rw26.c:916:ll_write_end()) page@000000002b59293a[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.926044:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000004e85cef 17ffffc0001001 3:0 ffff8b23ac4dc700 271 lru 00000080:00200000:1.0:1775764496.926048:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000d76c2933 271: 1< 0 - - > 2< 1110016 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.926051:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000002b59293a 00000080:00200000:1.0:1775764496.926432:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 16. 00000080:00200000:1.0:1775764496.926438:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 272 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.927725:0:51151:0:(rw26.c:858:ll_write_end()) pos 1114112, len 4096, copied 4096 00000080:00200000:1.0:1775764496.927740:0:51151:0:(rw26.c:916:ll_write_end()) page@000000000d39148c[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.927744:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000001c79f812 17ffffc0001001 3:0 ffff8b23ac4dcc40 272 lru 00000080:00200000:1.0:1775764496.927751:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000d93ea566 272: 1< 0 - - > 2< 1114112 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.927753:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000000d39148c 00000080:00200000:1.0:1775764496.927754:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 17. 00000080:00200000:1.0:1775764496.927759:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 273 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.927789:0:51151:0:(rw26.c:858:ll_write_end()) pos 1118208, len 4096, copied 4096 00000080:00200000:1.0:1775764496.928703:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000006674752[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.928717:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000da660013 17ffffc0001001 3:0 ffff8b23ac4dce00 273 lru 00000080:00200000:1.0:1775764496.928725:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000007ec22ad2 273: 1< 0 - - > 2< 1118208 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.928727:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000006674752 00000080:00200000:1.0:1775764496.928729:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 18. 00000080:00200000:1.0:1775764496.928732:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 274 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.928756:0:51151:0:(rw26.c:858:ll_write_end()) pos 1122304, len 4096, copied 4096 00000080:00200000:1.0:1775764496.928760:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000b3375bdb[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.928763:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000006c7d8073 17ffffc0001001 3:0 ffff8b23ac4dcb60 274 lru 00000080:00200000:1.0:1775764496.928768:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000e6173311 274: 1< 0 - - > 2< 1122304 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.928770:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000b3375bdb 00000080:00200000:1.0:1775764496.928771:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 19. 00000080:00200000:1.0:1775764496.928773:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 275 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.928786:0:51151:0:(rw26.c:858:ll_write_end()) pos 1126400, len 4096, copied 4096 00000080:00200000:1.0:1775764496.928789:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000f52db707[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.928791:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000711d8a57 17ffffc0001001 3:0 ffff8b23ac4dc9a0 275 lru 00000080:00200000:1.0:1775764496.930337:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000027d7ad6a 275: 1< 0 - - > 2< 1126400 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930346:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000f52db707 00000080:00200000:1.0:1775764496.930348:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 20. 00000080:00200000:1.0:1775764496.930356:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 276 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930410:0:51151:0:(rw26.c:858:ll_write_end()) pos 1130496, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930416:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000aea50ba4[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930420:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000005654e215 17ffffc0001001 3:0 ffff8b23ac4dc620 276 lru 00000080:00200000:1.0:1775764496.930426:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000616df952 276: 1< 0 - - > 2< 1130496 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930429:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000aea50ba4 00000080:00200000:1.0:1775764496.930430:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 21. 00000080:00200000:1.0:1775764496.930432:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 277 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930446:0:51151:0:(rw26.c:858:ll_write_end()) pos 1134592, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930449:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000082b724bf[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930467:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000cc2e79d8 17ffffc0001001 3:0 ffff8b23ac4dc000 277 lru 00000080:00200000:1.0:1775764496.930472:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000f5d1fa27 277: 1< 0 - - > 2< 1134592 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930474:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000082b724bf 00000080:00200000:1.0:1775764496.930476:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 22. 00000080:00200000:1.0:1775764496.930479:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 278 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930495:0:51151:0:(rw26.c:858:ll_write_end()) pos 1138688, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930499:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000fa5a2dfe[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930502:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000003def1640 17ffffc0001001 3:0 ffff8b23ac4dc8c0 278 lru 00000080:00200000:1.0:1775764496.930507:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000006a165795 278: 1< 0 - - > 2< 1138688 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930509:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000fa5a2dfe 00000080:00200000:1.0:1775764496.930510:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 23. 00000080:00200000:1.0:1775764496.930512:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 279 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930523:0:51151:0:(rw26.c:858:ll_write_end()) pos 1142784, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930526:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000a00c33e8[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930529:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000822308c8 17ffffc0001001 3:0 ffff8b23ac4dc2a0 279 lru 00000080:00200000:1.0:1775764496.930534:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000c5c86c54 279: 1< 0 - - > 2< 1142784 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930536:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000a00c33e8 00000080:00200000:1.0:1775764496.930537:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 24. 00000080:00200000:1.0:1775764496.930539:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 280 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930550:0:51151:0:(rw26.c:858:ll_write_end()) pos 1146880, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930554:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000bd573ef0[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930556:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000c9e18185 17ffffc0001001 3:0 ffff8b23ac4dc0e0 280 lru 00000080:00200000:1.0:1775764496.930560:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000f1a8491b 280: 1< 0 - - > 2< 1146880 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930563:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000bd573ef0 00000080:00200000:1.0:1775764496.930564:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 25. 00000080:00200000:1.0:1775764496.930565:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 281 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930577:0:51151:0:(rw26.c:858:ll_write_end()) pos 1150976, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930580:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000010555211[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930583:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000c8743478 17ffffc0001001 3:0 ffff8b23ac4dc460 281 lru 00000080:00200000:1.0:1775764496.930588:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000718e4d63 281: 1< 0 - - > 2< 1150976 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930590:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000010555211 00000080:00200000:1.0:1775764496.930591:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 26. 00000080:00200000:1.0:1775764496.930593:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 282 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930631:0:51151:0:(rw26.c:858:ll_write_end()) pos 1155072, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930636:0:51151:0:(rw26.c:916:ll_write_end()) page@000000009ca5d217[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930639:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000e07b149e 17ffffc0001021 2:0 ffff8b23ac4dc380 282 lru 00000080:00200000:1.0:1775764496.930645:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000034ec7929 282: 1< 0 - - > 2< 1155072 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930647:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000009ca5d217 00000080:00200000:1.0:1775764496.930648:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 27. 00000080:00200000:1.0:1775764496.930650:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 283 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930664:0:51151:0:(rw26.c:858:ll_write_end()) pos 1159168, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930668:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000078478c19[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930670:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000006809f4db 17ffffc0001001 3:0 ffff8b23ac4dcd20 283 lru 00000080:00200000:1.0:1775764496.930675:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000de4b094a 283: 1< 0 - - > 2< 1159168 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930677:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000078478c19 00000080:00200000:1.0:1775764496.930678:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 28. 00000080:00200000:1.0:1775764496.930680:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 284 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930695:0:51151:0:(rw26.c:858:ll_write_end()) pos 1163264, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930699:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000021fb87cb[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930701:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000ffc77947 17ffffc0001001 3:0 ffff8b23ac4dc7e0 284 lru 00000080:00200000:1.0:1775764496.930706:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000b5315c06 284: 1< 0 - - > 2< 1163264 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930708:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000021fb87cb 00000080:00200000:1.0:1775764496.930720:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 29. 00000080:00200000:1.0:1775764496.930722:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 285 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930745:0:51151:0:(rw26.c:858:ll_write_end()) pos 1167360, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930749:0:51151:0:(rw26.c:916:ll_write_end()) page@000000006945471c[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930752:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000c0f6cadf 17ffffc0001001 3:0 ffff8b23b9e232a0 285 lru 00000080:00200000:1.0:1775764496.930758:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000ba1039c2 285: 1< 0 - - > 2< 1167360 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930760:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000006945471c 00000080:00200000:1.0:1775764496.930761:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 30. 00000080:00200000:1.0:1775764496.930763:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 286 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930777:0:51151:0:(rw26.c:858:ll_write_end()) pos 1171456, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930781:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000a9693f9e[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930784:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000dc2bae62 17ffffc0001001 3:0 ffff8b23b9e238c0 286 lru 00000080:00200000:1.0:1775764496.930789:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000b9e82b17 286: 1< 0 - - > 2< 1171456 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930791:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000a9693f9e 00000080:00200000:1.0:1775764496.930792:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 31. 00000080:00200000:1.0:1775764496.930795:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 287 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930806:0:51151:0:(rw26.c:858:ll_write_end()) pos 1175552, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930810:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000fc11061b[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930812:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000a0fd1616 17ffffc0001001 3:0 ffff8b23b9e23000 287 lru 00000080:00200000:1.0:1775764496.930817:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000a714f77b 287: 1< 0 - - > 2< 1175552 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930819:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000fc11061b 00000080:00200000:1.0:1775764496.930820:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 32. 00000080:00200000:1.0:1775764496.930822:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 288 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930833:0:51151:0:(rw26.c:858:ll_write_end()) pos 1179648, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930837:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000018e69dc5[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930839:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000b4739e3b 17ffffc0001001 3:0 ffff8b23b9e23620 288 lru 00000080:00200000:1.0:1775764496.930844:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000e4979712 288: 1< 0 - - > 2< 1179648 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930847:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000018e69dc5 00000080:00200000:1.0:1775764496.930847:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 33. 00000080:00200000:1.0:1775764496.930849:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 289 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930861:0:51151:0:(rw26.c:858:ll_write_end()) pos 1183744, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930865:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000085e2b273[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930868:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000005d288f06 17ffffc0001001 3:0 ffff8b23b9e239a0 289 lru 00000080:00200000:1.0:1775764496.930873:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000004b7b9e4 289: 1< 0 - - > 2< 1183744 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930875:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000085e2b273 00000080:00200000:1.0:1775764496.930876:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 34. 00000080:00200000:1.0:1775764496.930878:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 290 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930890:0:51151:0:(rw26.c:858:ll_write_end()) pos 1187840, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930894:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000e5e482bf[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930897:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000ab1d50de 17ffffc0001001 3:0 ffff8b23b9e23b60 290 lru 00000080:00200000:1.0:1775764496.930903:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000e4bf6c96 290: 1< 0 - - > 2< 1187840 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930905:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000e5e482bf 00000080:00200000:1.0:1775764496.930906:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 35. 00000080:00200000:1.0:1775764496.930908:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 291 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930923:0:51151:0:(rw26.c:858:ll_write_end()) pos 1191936, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930927:0:51151:0:(rw26.c:916:ll_write_end()) page@000000006772140e[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930929:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000003bfc3c6c 17ffffc0001001 3:0 ffff8b23b9e23e00 291 lru 00000080:00200000:1.0:1775764496.930935:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000087566613 291: 1< 0 - - > 2< 1191936 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930938:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000006772140e 00000080:00200000:1.0:1775764496.930939:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 36. 00000080:00200000:1.0:1775764496.930941:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 292 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930954:0:51151:0:(rw26.c:858:ll_write_end()) pos 1196032, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930958:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000199591bc[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930960:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000006cc04e6b 17ffffc0001001 3:0 ffff8b23b9e23c40 292 lru 00000080:00200000:1.0:1775764496.930964:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000008aba84f6 292: 1< 0 - - > 2< 1196032 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930966:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000199591bc 00000080:00200000:1.0:1775764496.930968:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 37. 00000080:00200000:1.0:1775764496.930969:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 293 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.930984:0:51151:0:(rw26.c:858:ll_write_end()) pos 1200128, len 4096, copied 4096 00000080:00200000:1.0:1775764496.930988:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000b6803f9f[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.930991:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000227accec 17ffffc0001001 3:0 ffff8b23b9e23a80 293 lru 00000080:00200000:1.0:1775764496.930996:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000073bbf47f 293: 1< 0 - - > 2< 1200128 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.930998:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000b6803f9f 00000080:00200000:1.0:1775764496.930999:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 38. 00000080:00200000:1.0:1775764496.931001:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 294 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931015:0:51151:0:(rw26.c:858:ll_write_end()) pos 1204224, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931019:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000c2d4d555[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931021:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000825477bd 17ffffc0001001 3:0 ffff8b23b9e23ee0 294 lru 00000080:00200000:1.0:1775764496.931027:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000df3e0227 294: 1< 0 - - > 2< 1204224 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931029:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000c2d4d555 00000080:00200000:1.0:1775764496.931030:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 39. 00000080:00200000:1.0:1775764496.931032:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 295 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931041:0:51151:0:(rw26.c:858:ll_write_end()) pos 1208320, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931044:0:51151:0:(rw26.c:916:ll_write_end()) page@000000004b47ca74[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931047:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000009c986da2 17ffffc0001001 3:0 ffff8b23b9e231c0 295 lru 00000080:00200000:1.0:1775764496.931052:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000c3f53fa1 295: 1< 0 - - > 2< 1208320 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931055:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000004b47ca74 00000080:00200000:1.0:1775764496.931055:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 40. 00000080:00200000:1.0:1775764496.931057:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 296 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931070:0:51151:0:(rw26.c:858:ll_write_end()) pos 1212416, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931073:0:51151:0:(rw26.c:916:ll_write_end()) page@000000004f6026ef[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931076:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000d478aa64 17ffffc0001001 3:0 ffff8b23b9e23540 296 lru 00000080:00200000:1.0:1775764496.931081:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000ea16d00a 296: 1< 0 - - > 2< 1212416 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931083:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000004f6026ef 00000080:00200000:1.0:1775764496.931084:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 41. 00000080:00200000:1.0:1775764496.931086:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 297 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931102:0:51151:0:(rw26.c:858:ll_write_end()) pos 1216512, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931106:0:51151:0:(rw26.c:916:ll_write_end()) page@000000009201c4da[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931108:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000181594b3 17ffffc0001021 2:0 ffff8b23b9e23700 297 lru 00000080:00200000:1.0:1775764496.931114:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000002771cb7b 297: 1< 0 - - > 2< 1216512 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931116:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000009201c4da 00000080:00200000:1.0:1775764496.931117:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 42. 00000080:00200000:1.0:1775764496.931119:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 298 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931131:0:51151:0:(rw26.c:858:ll_write_end()) pos 1220608, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931134:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000a2fd54bd[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931137:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000043c48fa8 17ffffc0001001 3:0 ffff8b23b9e237e0 298 lru 00000080:00200000:1.0:1775764496.931142:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000003e2ce6d7 298: 1< 0 - - > 2< 1220608 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931144:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000a2fd54bd 00000080:00200000:1.0:1775764496.931145:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 43. 00000080:00200000:1.0:1775764496.931147:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 299 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931156:0:51151:0:(rw26.c:858:ll_write_end()) pos 1224704, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931160:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000f4e54458[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931163:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000082a02c6f 17ffffc0001001 3:0 ffff8b23b9e23d20 299 lru 00000080:00200000:1.0:1775764496.931168:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000005e8b2dfe 299: 1< 0 - - > 2< 1224704 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931170:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000f4e54458 00000080:00200000:1.0:1775764496.931171:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 44. 00000080:00200000:1.0:1775764496.931173:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 300 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931187:0:51151:0:(rw26.c:858:ll_write_end()) pos 1228800, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931191:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000028603973[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931194:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000a98feaff 17ffffc0001001 3:0 ffff8b23b9e23380 300 lru 00000080:00200000:1.0:1775764496.931200:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000073b69931 300: 1< 0 - - > 2< 1228800 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931202:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000028603973 00000080:00200000:1.0:1775764496.931203:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 45. 00000080:00200000:1.0:1775764496.931205:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 301 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931215:0:51151:0:(rw26.c:858:ll_write_end()) pos 1232896, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931219:0:51151:0:(rw26.c:916:ll_write_end()) page@000000002e6f0c06[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931221:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000b1aee3c4 17ffffc0001001 3:0 ffff8b23b9e23460 301 lru 00000080:00200000:1.0:1775764496.931227:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000002d65030f 301: 1< 0 - - > 2< 1232896 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931228:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000002e6f0c06 00000080:00200000:1.0:1775764496.931230:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 46. 00000080:00200000:1.0:1775764496.931231:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 302 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931243:0:51151:0:(rw26.c:858:ll_write_end()) pos 1236992, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931246:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000089406862[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931249:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000a528b3f5 17ffffc0001001 3:0 ffff8b23b9e230e0 302 lru 00000080:00200000:1.0:1775764496.931254:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000fd406652 302: 1< 0 - - > 2< 1236992 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931256:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000089406862 00000080:00200000:1.0:1775764496.931257:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 47. 00000080:00200000:1.0:1775764496.931259:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 303 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931275:0:51151:0:(rw26.c:858:ll_write_end()) pos 1241088, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931278:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000088730ffc[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931281:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000bb2db720 17ffffc0001001 3:0 ffff8b2391951000 303 lru 00000080:00200000:1.0:1775764496.931286:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000ba439deb 303: 1< 0 - - > 2< 1241088 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931288:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000088730ffc 00000080:00200000:1.0:1775764496.931289:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 48. 00000080:00200000:1.0:1775764496.931291:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 304 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931303:0:51151:0:(rw26.c:858:ll_write_end()) pos 1245184, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931307:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000069a5689d[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931309:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000b15e6360 17ffffc0001001 3:0 ffff8b2391951620 304 lru 00000080:00200000:1.0:1775764496.931315:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000001e1205f5 304: 1< 0 - - > 2< 1245184 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931317:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000069a5689d 00000080:00200000:1.0:1775764496.931318:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 49. 00000080:00200000:1.0:1775764496.931321:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 305 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931334:0:51151:0:(rw26.c:858:ll_write_end()) pos 1249280, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931338:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000024903771[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931340:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000008a0b3d24 17ffffc0001001 3:0 ffff8b23919519a0 305 lru 00000080:00200000:1.0:1775764496.931345:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000dd640d21 305: 1< 0 - - > 2< 1249280 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931347:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000024903771 00000080:00200000:1.0:1775764496.931348:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 50. 00000080:00200000:1.0:1775764496.931350:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 306 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931363:0:51151:0:(rw26.c:858:ll_write_end()) pos 1253376, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931367:0:51151:0:(rw26.c:916:ll_write_end()) page@000000005df471fb[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931370:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000165ed692 17ffffc0001001 3:0 ffff8b2391951b60 306 lru 00000080:00200000:1.0:1775764496.931375:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000078cb2ee8 306: 1< 0 - - > 2< 1253376 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931377:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000005df471fb 00000080:00200000:1.0:1775764496.931378:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 51. 00000080:00200000:1.0:1775764496.931380:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 307 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931390:0:51151:0:(rw26.c:858:ll_write_end()) pos 1257472, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931393:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000eff6aa36[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931395:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000075700141 17ffffc0001001 3:0 ffff8b2391951e00 307 lru 00000080:00200000:1.0:1775764496.931401:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000a851d4fd 307: 1< 0 - - > 2< 1257472 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931403:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000eff6aa36 00000080:00200000:1.0:1775764496.931404:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 52. 00000080:00200000:1.0:1775764496.931405:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 308 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931415:0:51151:0:(rw26.c:858:ll_write_end()) pos 1261568, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931418:0:51151:0:(rw26.c:916:ll_write_end()) page@000000003ee88125[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931420:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000003ac9b2be 17ffffc0001001 3:0 ffff8b2391951c40 308 lru 00000080:00200000:1.0:1775764496.931426:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000083156855 308: 1< 0 - - > 2< 1261568 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931428:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000003ee88125 00000080:00200000:1.0:1775764496.931429:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 53. 00000080:00200000:1.0:1775764496.931430:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 309 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931440:0:51151:0:(rw26.c:858:ll_write_end()) pos 1265664, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931443:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000cb158e31[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931445:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000cbb97c8b 17ffffc0001001 3:0 ffff8b2391951a80 309 lru 00000080:00200000:1.0:1775764496.931483:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000003ee4d2fa 309: 1< 0 - - > 2< 1265664 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931486:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000cb158e31 00000080:00200000:1.0:1775764496.931487:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 54. 00000080:00200000:1.0:1775764496.931490:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 310 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931506:0:51151:0:(rw26.c:858:ll_write_end()) pos 1269760, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931510:0:51151:0:(rw26.c:916:ll_write_end()) page@000000005ec62f40[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931512:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000fd2ed6ef 17ffffc0001001 3:0 ffff8b2391951ee0 310 lru 00000080:00200000:1.0:1775764496.931518:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000001c582df4 310: 1< 0 - - > 2< 1269760 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931520:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000005ec62f40 00000080:00200000:1.0:1775764496.931521:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 55. 00000080:00200000:1.0:1775764496.931523:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 311 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931534:0:51151:0:(rw26.c:858:ll_write_end()) pos 1273856, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931537:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000051096da8[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931540:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000014cdedc3 17ffffc0001001 3:0 ffff8b23919511c0 311 lru 00000080:00200000:1.0:1775764496.931545:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000003cae1375 311: 1< 0 - - > 2< 1273856 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931547:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000051096da8 00000080:00200000:1.0:1775764496.931548:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 56. 00000080:00200000:1.0:1775764496.931550:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 312 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931569:0:51151:0:(rw26.c:858:ll_write_end()) pos 1277952, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931573:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000034a60829[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931576:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000e92bc8ec 17ffffc0001021 2:0 ffff8b2391951540 312 lru 00000080:00200000:1.0:1775764496.931581:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000035a18ffa 312: 1< 0 - - > 2< 1277952 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931584:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000034a60829 00000080:00200000:1.0:1775764496.931585:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 57. 00000080:00200000:1.0:1775764496.931587:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 313 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931601:0:51151:0:(rw26.c:858:ll_write_end()) pos 1282048, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931604:0:51151:0:(rw26.c:916:ll_write_end()) page@000000005f8a2662[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931607:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000b8597a2e 17ffffc0001001 3:0 ffff8b2391951700 313 lru 00000080:00200000:1.0:1775764496.931613:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000002f0e4a21 313: 1< 0 - - > 2< 1282048 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931615:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000005f8a2662 00000080:00200000:1.0:1775764496.931616:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 58. 00000080:00200000:1.0:1775764496.931618:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 314 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931629:0:51151:0:(rw26.c:858:ll_write_end()) pos 1286144, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931633:0:51151:0:(rw26.c:916:ll_write_end()) page@000000001b68add1[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931635:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000056864d00 17ffffc0001001 3:0 ffff8b23919517e0 314 lru 00000080:00200000:1.0:1775764496.931640:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000c05f25c0 314: 1< 0 - - > 2< 1286144 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931642:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000001b68add1 00000080:00200000:1.0:1775764496.931643:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 59. 00000080:00200000:1.0:1775764496.931645:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 315 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931656:0:51151:0:(rw26.c:858:ll_write_end()) pos 1290240, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931660:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000133c245d[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931663:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000df75828e 17ffffc0001001 3:0 ffff8b2391951d20 315 lru 00000080:00200000:1.0:1775764496.931668:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000f5675a52 315: 1< 0 - - > 2< 1290240 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931670:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000133c245d 00000080:00200000:1.0:1775764496.931671:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 60. 00000080:00200000:1.0:1775764496.931673:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 316 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931683:0:51151:0:(rw26.c:858:ll_write_end()) pos 1294336, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931686:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000433dd7a4[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931688:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000e7059900 17ffffc0001001 3:0 ffff8b2391951380 316 lru 00000080:00200000:1.0:1775764496.931692:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000006464d14 316: 1< 0 - - > 2< 1294336 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931695:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000433dd7a4 00000080:00200000:1.0:1775764496.931696:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 61. 00000080:00200000:1.0:1775764496.931697:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 317 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931722:0:51151:0:(rw26.c:858:ll_write_end()) pos 1298432, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931727:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000050a4cd3e[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931729:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000024e0b685 17ffffc0001001 3:0 ffff8b2391951460 317 lru 00000080:00200000:1.0:1775764496.931733:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000001dc39ee8 317: 1< 0 - - > 2< 1298432 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931735:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000050a4cd3e 00000080:00200000:1.0:1775764496.931736:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 62. 00000080:00200000:1.0:1775764496.931738:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 318 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931752:0:51151:0:(rw26.c:858:ll_write_end()) pos 1302528, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931755:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000f2f7797a[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931757:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000004c53f934 17ffffc0001001 3:0 ffff8b23919510e0 318 lru 00000080:00200000:1.0:1775764496.931762:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000076b836de 318: 1< 0 - - > 2< 1302528 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931764:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000f2f7797a 00000080:00200000:1.0:1775764496.931765:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 63. 00000080:00200000:1.0:1775764496.931767:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 319 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931782:0:51151:0:(rw26.c:858:ll_write_end()) pos 1306624, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931785:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000033f47411[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931787:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000008bfd0787 17ffffc0001001 3:0 ffff8b23919512a0 319 lru 00000080:00200000:1.0:1775764496.931791:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000d2be0609 319: 1< 0 - - > 2< 1306624 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931794:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000033f47411 00000080:00200000:1.0:1775764496.931795:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 64. 00000080:00200000:1.0:1775764496.931797:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 320 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931815:0:51151:0:(rw26.c:858:ll_write_end()) pos 1310720, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931819:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000f0bbff20[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931822:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000e2ec46fc 17ffffc0001001 3:0 ffff8b23919518c0 320 lru 00000080:00200000:1.0:1775764496.931826:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000439a07e6 320: 1< 0 - - > 2< 1310720 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931828:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000f0bbff20 00000080:00200000:1.0:1775764496.931829:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 65. 00000080:00200000:1.0:1775764496.931831:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 321 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931846:0:51151:0:(rw26.c:858:ll_write_end()) pos 1314816, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931850:0:51151:0:(rw26.c:916:ll_write_end()) page@000000002def5aac[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931853:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000bba1364c 17ffffc0001001 3:0 ffff8b23a78172a0 321 lru 00000080:00200000:1.0:1775764496.931857:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000006d18ff6b 321: 1< 0 - - > 2< 1314816 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931859:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000002def5aac 00000080:00200000:1.0:1775764496.931860:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 66. 00000080:00200000:1.0:1775764496.931862:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 322 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931877:0:51151:0:(rw26.c:858:ll_write_end()) pos 1318912, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931880:0:51151:0:(rw26.c:916:ll_write_end()) page@000000006538c7fe[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931882:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000f19f802d 17ffffc0001001 3:0 ffff8b23a78178c0 322 lru 00000080:00200000:1.0:1775764496.931887:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000063f992aa 322: 1< 0 - - > 2< 1318912 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931889:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000006538c7fe 00000080:00200000:1.0:1775764496.931890:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 67. 00000080:00200000:1.0:1775764496.931892:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 323 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931904:0:51151:0:(rw26.c:858:ll_write_end()) pos 1323008, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931907:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000edfd37ce[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931909:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000ba0f5f6f 17ffffc0001001 3:0 ffff8b23a7817000 323 lru 00000080:00200000:1.0:1775764496.931914:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000dddcaa51 323: 1< 0 - - > 2< 1323008 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931916:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000edfd37ce 00000080:00200000:1.0:1775764496.931917:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 68. 00000080:00200000:1.0:1775764496.931919:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 324 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931931:0:51151:0:(rw26.c:858:ll_write_end()) pos 1327104, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931935:0:51151:0:(rw26.c:916:ll_write_end()) page@000000003816b34b[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931938:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000015f749e4 17ffffc0001001 3:0 ffff8b23a7817620 324 lru 00000080:00200000:1.0:1775764496.931944:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000a9ac72df 324: 1< 0 - - > 2< 1327104 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931946:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000003816b34b 00000080:00200000:1.0:1775764496.931947:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 69. 00000080:00200000:1.0:1775764496.931950:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 325 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931963:0:51151:0:(rw26.c:858:ll_write_end()) pos 1331200, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931966:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000b47c39b2[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931969:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000003433dd7a 17ffffc0001001 3:0 ffff8b23a78179a0 325 lru 00000080:00200000:1.0:1775764496.931973:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000ecd5676f 325: 1< 0 - - > 2< 1331200 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.931975:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000b47c39b2 00000080:00200000:1.0:1775764496.931976:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 70. 00000080:00200000:1.0:1775764496.931978:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 326 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.931992:0:51151:0:(rw26.c:858:ll_write_end()) pos 1335296, len 4096, copied 4096 00000080:00200000:1.0:1775764496.931995:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000e1395c40[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.931997:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000c7ed972a 17ffffc0001001 3:0 ffff8b23a7817b60 326 lru 00000080:00200000:1.0:1775764496.932002:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000c90d0d2c 326: 1< 0 - - > 2< 1335296 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932004:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000e1395c40 00000080:00200000:1.0:1775764496.932005:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 71. 00000080:00200000:1.0:1775764496.932007:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 327 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932023:0:51151:0:(rw26.c:858:ll_write_end()) pos 1339392, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932027:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000625bdbb9[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932029:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000f3aa3446 17ffffc0001021 2:0 ffff8b23a7817e00 327 lru 00000080:00200000:1.0:1775764496.932034:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000f30ccf45 327: 1< 0 - - > 2< 1339392 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932035:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000625bdbb9 00000080:00200000:1.0:1775764496.932036:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 72. 00000080:00200000:1.0:1775764496.932038:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 328 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932051:0:51151:0:(rw26.c:858:ll_write_end()) pos 1343488, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932054:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000057462ecc[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932056:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000000953d968 17ffffc0001001 3:0 ffff8b23a7817c40 328 lru 00000080:00200000:1.0:1775764496.932060:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000001179ad29 328: 1< 0 - - > 2< 1343488 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932062:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000057462ecc 00000080:00200000:1.0:1775764496.932063:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 73. 00000080:00200000:1.0:1775764496.932064:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 329 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932075:0:51151:0:(rw26.c:858:ll_write_end()) pos 1347584, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932078:0:51151:0:(rw26.c:916:ll_write_end()) page@000000002eda2b5c[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932080:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000bb2105fd 17ffffc0001001 3:0 ffff8b23a7817a80 329 lru 00000080:00200000:1.0:1775764496.932085:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000c36683c0 329: 1< 0 - - > 2< 1347584 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932087:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000002eda2b5c 00000080:00200000:1.0:1775764496.932088:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 74. 00000080:00200000:1.0:1775764496.932089:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 330 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932101:0:51151:0:(rw26.c:858:ll_write_end()) pos 1351680, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932104:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000060102ccc[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932107:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000390fe468 17ffffc0001001 3:0 ffff8b23a7817ee0 330 lru 00000080:00200000:1.0:1775764496.932112:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000007c3c7471 330: 1< 0 - - > 2< 1351680 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932114:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000060102ccc 00000080:00200000:1.0:1775764496.932114:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 75. 00000080:00200000:1.0:1775764496.932116:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 331 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932129:0:51151:0:(rw26.c:858:ll_write_end()) pos 1355776, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932132:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000519da5fa[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932135:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000095d0fd80 17ffffc0001001 3:0 ffff8b23a78171c0 331 lru 00000080:00200000:1.0:1775764496.932139:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000006db12ff4 331: 1< 0 - - > 2< 1355776 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932141:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000519da5fa 00000080:00200000:1.0:1775764496.932142:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 76. 00000080:00200000:1.0:1775764496.932144:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 332 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932155:0:51151:0:(rw26.c:858:ll_write_end()) pos 1359872, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932158:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000039906247[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932160:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000cdee73c2 17ffffc0001001 3:0 ffff8b23a7817540 332 lru 00000080:00200000:1.0:1775764496.932164:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000f70d0efd 332: 1< 0 - - > 2< 1359872 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932166:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000039906247 00000080:00200000:1.0:1775764496.932167:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 77. 00000080:00200000:1.0:1775764496.932169:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 333 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932180:0:51151:0:(rw26.c:858:ll_write_end()) pos 1363968, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932183:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000b9afe532[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932185:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000001b55131a 17ffffc0001001 3:0 ffff8b23a7817700 333 lru 00000080:00200000:1.0:1775764496.932190:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000112400cd 333: 1< 0 - - > 2< 1363968 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932191:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000b9afe532 00000080:00200000:1.0:1775764496.932192:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 78. 00000080:00200000:1.0:1775764496.932194:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 334 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932205:0:51151:0:(rw26.c:858:ll_write_end()) pos 1368064, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932208:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000c5938da5[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932210:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000b14741a8 17ffffc0001001 3:0 ffff8b23a78177e0 334 lru 00000080:00200000:1.0:1775764496.932215:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000007f4de56f 334: 1< 0 - - > 2< 1368064 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932217:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000c5938da5 00000080:00200000:1.0:1775764496.932217:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 79. 00000080:00200000:1.0:1775764496.932219:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 335 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932232:0:51151:0:(rw26.c:858:ll_write_end()) pos 1372160, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932235:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000656efd4a[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932238:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000179f1510 17ffffc0001001 3:0 ffff8b23a7817d20 335 lru 00000080:00200000:1.0:1775764496.932242:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000004236e868 335: 1< 0 - - > 2< 1372160 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932244:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000656efd4a 00000080:00200000:1.0:1775764496.932245:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 80. 00000080:00200000:1.0:1775764496.932247:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 336 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932262:0:51151:0:(rw26.c:858:ll_write_end()) pos 1376256, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932266:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000184520b9[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932268:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000e361b50a 17ffffc0001001 3:0 ffff8b23a7817380 336 lru 00000080:00200000:1.0:1775764496.932273:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000f9f26ab0 336: 1< 0 - - > 2< 1376256 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932276:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000184520b9 00000080:00200000:1.0:1775764496.932276:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 81. 00000080:00200000:1.0:1775764496.932278:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 337 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932291:0:51151:0:(rw26.c:858:ll_write_end()) pos 1380352, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932294:0:51151:0:(rw26.c:916:ll_write_end()) page@000000009e87b90a[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932297:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000ea28dfef 17ffffc0001001 3:0 ffff8b23a7817460 337 lru 00000080:00200000:1.0:1775764496.932302:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000006665f625 337: 1< 0 - - > 2< 1380352 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932304:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000009e87b90a 00000080:00200000:1.0:1775764496.932305:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 82. 00000080:00200000:1.0:1775764496.932307:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 338 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932319:0:51151:0:(rw26.c:858:ll_write_end()) pos 1384448, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932321:0:51151:0:(rw26.c:916:ll_write_end()) page@000000009ced454a[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932324:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000258dbdea 17ffffc0001001 3:0 ffff8b23a78170e0 338 lru 00000080:00200000:1.0:1775764496.932328:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000796dce7e 338: 1< 0 - - > 2< 1384448 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932330:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000009ced454a 00000080:00200000:1.0:1775764496.932331:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 83. 00000080:00200000:1.0:1775764496.932333:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 339 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932344:0:51151:0:(rw26.c:858:ll_write_end()) pos 1388544, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932347:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000064340a1d[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932349:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000ec55c3e6 17ffffc0001001 3:0 ffff8b23b9eb00e0 339 lru 00000080:00200000:1.0:1775764496.932354:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000d4ba46e2 339: 1< 0 - - > 2< 1388544 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932357:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000064340a1d 00000080:00200000:1.0:1775764496.932358:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 84. 00000080:00200000:1.0:1775764496.932359:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 340 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932381:0:51151:0:(rw26.c:858:ll_write_end()) pos 1392640, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932385:0:51151:0:(rw26.c:916:ll_write_end()) page@000000009f9f5df5[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932388:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000090df5811 17ffffc0001001 3:0 ffff8b23b9eb02a0 340 lru 00000080:00200000:1.0:1775764496.932394:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000075161929 340: 1< 0 - - > 2< 1392640 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932396:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000009f9f5df5 00000080:00200000:1.0:1775764496.932397:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 85. 00000080:00200000:1.0:1775764496.932399:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 341 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932416:0:51151:0:(rw26.c:858:ll_write_end()) pos 1396736, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932420:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000047672d0c[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932422:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000000f1b1cce 17ffffc0001001 3:0 ffff8b23b9eb08c0 341 lru 00000080:00200000:1.0:1775764496.932426:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000c7eaa22c 341: 1< 0 - - > 2< 1396736 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932428:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000047672d0c 00000080:00200000:1.0:1775764496.932429:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 86. 00000080:00200000:1.0:1775764496.932431:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 342 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932445:0:51151:0:(rw26.c:858:ll_write_end()) pos 1400832, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932449:0:51151:0:(rw26.c:916:ll_write_end()) page@000000006cbd3444[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932481:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000ea4557b5 17ffffc0001021 2:0 ffff8b23b9eb0000 342 lru 00000080:00200000:1.0:1775764496.932488:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000efe10c33 342: 1< 0 - - > 2< 1400832 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932490:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000006cbd3444 00000080:00200000:1.0:1775764496.932491:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 87. 00000080:00200000:1.0:1775764496.932493:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 343 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932508:0:51151:0:(rw26.c:858:ll_write_end()) pos 1404928, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932512:0:51151:0:(rw26.c:916:ll_write_end()) page@000000003c4abfe2[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932514:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000497075b4 17ffffc0001001 3:0 ffff8b23b9eb0620 343 lru 00000080:00200000:1.0:1775764496.932519:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000015f051d2 343: 1< 0 - - > 2< 1404928 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932521:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000003c4abfe2 00000080:00200000:1.0:1775764496.932522:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 88. 00000080:00200000:1.0:1775764496.932523:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 344 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932534:0:51151:0:(rw26.c:858:ll_write_end()) pos 1409024, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932539:0:51151:0:(rw26.c:916:ll_write_end()) page@000000003510de47[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932541:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000773901aa 17ffffc0001001 3:0 ffff8b23b9eb09a0 344 lru 00000080:00200000:1.0:1775764496.932546:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000049b98c97 344: 1< 0 - - > 2< 1409024 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932548:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000003510de47 00000080:00200000:1.0:1775764496.932549:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 89. 00000080:00200000:1.0:1775764496.932550:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 345 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932564:0:51151:0:(rw26.c:858:ll_write_end()) pos 1413120, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932567:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000076ae33b3[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932570:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000fc92d656 17ffffc0001001 3:0 ffff8b23b9eb0460 345 lru 00000080:00200000:1.0:1775764496.932574:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000022e9f6f 345: 1< 0 - - > 2< 1413120 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932576:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000076ae33b3 00000080:00200000:1.0:1775764496.932577:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 90. 00000080:00200000:1.0:1775764496.932579:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 346 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932592:0:51151:0:(rw26.c:858:ll_write_end()) pos 1417216, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932596:0:51151:0:(rw26.c:916:ll_write_end()) page@000000003fde48cb[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932598:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000001ac71afc 17ffffc0001001 3:0 ffff8b23b9eb0380 346 lru 00000080:00200000:1.0:1775764496.932602:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000040faeab6 346: 1< 0 - - > 2< 1417216 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932603:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000003fde48cb 00000080:00200000:1.0:1775764496.932604:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 91. 00000080:00200000:1.0:1775764496.932606:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 347 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932619:0:51151:0:(rw26.c:858:ll_write_end()) pos 1421312, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932622:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000b376d7e7[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932625:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000e07eae16 17ffffc0001001 3:0 ffff8b23b9eb0d20 347 lru 00000080:00200000:1.0:1775764496.932629:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000007def3882 347: 1< 0 - - > 2< 1421312 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932631:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000b376d7e7 00000080:00200000:1.0:1775764496.932632:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 92. 00000080:00200000:1.0:1775764496.932633:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 348 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932646:0:51151:0:(rw26.c:858:ll_write_end()) pos 1425408, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932649:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000b0b58598[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932652:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000005ed69f53 17ffffc0001001 3:0 ffff8b23b9eb07e0 348 lru 00000080:00200000:1.0:1775764496.932656:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000e331b077 348: 1< 0 - - > 2< 1425408 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932658:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000b0b58598 00000080:00200000:1.0:1775764496.932660:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 93. 00000080:00200000:1.0:1775764496.932662:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 349 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932676:0:51151:0:(rw26.c:858:ll_write_end()) pos 1429504, len 4096, copied 4096 00000080:00200000:1.0:1775764496.932679:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000f835153b[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.932682:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000003e64a744 17ffffc0001001 3:0 ffff8b23b9eb0700 349 lru 00000080:00200000:1.0:1775764496.932687:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000e2b54fd1 349: 1< 0 - - > 2< 1429504 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.932689:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000f835153b 00000080:00200000:1.0:1775764496.932690:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 94. 00000080:00200000:1.0:1775764496.932692:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 350 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.932705:0:51151:0:(rw26.c:858:ll_write_end()) pos 1433600, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937212:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000849562f8[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937222:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000025c7d66c 17ffffc0001001 3:0 ffff8b23b9eb0540 350 lru 00000080:00200000:1.0:1775764496.937232:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000fb3e840e 350: 1< 0 - - > 2< 1433600 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937235:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000849562f8 00000080:00200000:1.0:1775764496.937238:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 95. 00000080:00200000:1.0:1775764496.937286:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 351 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937353:0:51151:0:(rw26.c:858:ll_write_end()) pos 1437696, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937360:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000077f4893c[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937362:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000000e95a7ca 17ffffc0001001 3:0 ffff8b23b9eb01c0 351 lru 00000080:00200000:1.0:1775764496.937367:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000007a79bc46 351: 1< 0 - - > 2< 1437696 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937370:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000077f4893c 00000080:00200000:1.0:1775764496.937371:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 96. 00000080:00200000:1.0:1775764496.937373:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 352 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937388:0:51151:0:(rw26.c:858:ll_write_end()) pos 1441792, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937392:0:51151:0:(rw26.c:916:ll_write_end()) page@000000002b376656[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937401:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000085d460c9 17ffffc0001001 3:0 ffff8b23b9eb0ee0 352 lru 00000080:00200000:1.0:1775764496.937407:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000009a305cf8 352: 1< 0 - - > 2< 1441792 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937409:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000002b376656 00000080:00200000:1.0:1775764496.937410:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 97. 00000080:00200000:1.0:1775764496.937412:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 353 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937425:0:51151:0:(rw26.c:858:ll_write_end()) pos 1445888, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937429:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000ae2467e2[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937432:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000d6b669a1 17ffffc0001001 3:0 ffff8b23b9eb0a80 353 lru 00000080:00200000:1.0:1775764496.937437:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000008332bc82 353: 1< 0 - - > 2< 1445888 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937439:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000ae2467e2 00000080:00200000:1.0:1775764496.937440:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 98. 00000080:00200000:1.0:1775764496.937441:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 354 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937476:0:51151:0:(rw26.c:858:ll_write_end()) pos 1449984, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937481:0:51151:0:(rw26.c:916:ll_write_end()) page@000000007939b9e2[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937484:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000f5af34f9 17ffffc0001001 3:0 ffff8b23b9eb0c40 354 lru 00000080:00200000:1.0:1775764496.937490:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000007999ae18 354: 1< 0 - - > 2< 1449984 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937492:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000007939b9e2 00000080:00200000:1.0:1775764496.937493:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 99. 00000080:00200000:1.0:1775764496.937495:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 355 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937509:0:51151:0:(rw26.c:858:ll_write_end()) pos 1454080, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937513:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000cebeb7d5[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937516:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000b07a441b 17ffffc0001001 3:0 ffff8b23b9eb0e00 355 lru 00000080:00200000:1.0:1775764496.937521:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000d1554de8 355: 1< 0 - - > 2< 1454080 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937523:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000cebeb7d5 00000080:00200000:1.0:1775764496.937525:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 100. 00000080:00200000:1.0:1775764496.937526:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 356 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937541:0:51151:0:(rw26.c:858:ll_write_end()) pos 1458176, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937545:0:51151:0:(rw26.c:916:ll_write_end()) page@000000004504b440[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937547:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000553049bf 17ffffc0001001 3:0 ffff8b23b9eb0b60 356 lru 00000080:00200000:1.0:1775764496.937553:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000003e15cf62 356: 1< 0 - - > 2< 1458176 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937554:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000004504b440 00000080:00200000:1.0:1775764496.937555:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 101. 00000080:00200000:1.0:1775764496.937557:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 357 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937581:0:51151:0:(rw26.c:858:ll_write_end()) pos 1462272, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937585:0:51151:0:(rw26.c:916:ll_write_end()) page@000000007cdc14d9[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937588:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000000b500c2c 17ffffc0001021 2:0 ffff8b23b9d84e00 357 lru 00000080:00200000:1.0:1775764496.937593:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000002e76d897 357: 1< 0 - - > 2< 1462272 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937607:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000007cdc14d9 00000080:00200000:1.0:1775764496.937609:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 102. 00000080:00200000:1.0:1775764496.937611:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 358 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937627:0:51151:0:(rw26.c:858:ll_write_end()) pos 1466368, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937631:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000b8f6019e[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937633:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000096d8dbca 17ffffc0001001 3:0 ffff8b23b9d84b60 358 lru 00000080:00200000:1.0:1775764496.937638:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000add73afa 358: 1< 0 - - > 2< 1466368 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937640:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000b8f6019e 00000080:00200000:1.0:1775764496.937641:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 103. 00000080:00200000:1.0:1775764496.937644:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 359 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937657:0:51151:0:(rw26.c:858:ll_write_end()) pos 1470464, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937660:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000c68f65e8[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937663:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000b29cf656 17ffffc0001001 3:0 ffff8b23b9d849a0 359 lru 00000080:00200000:1.0:1775764496.937667:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000006ece93ae 359: 1< 0 - - > 2< 1470464 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937669:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000c68f65e8 00000080:00200000:1.0:1775764496.937670:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 104. 00000080:00200000:1.0:1775764496.937672:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 360 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937686:0:51151:0:(rw26.c:858:ll_write_end()) pos 1474560, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937689:0:51151:0:(rw26.c:916:ll_write_end()) page@000000006373799f[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937691:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000043d124d5 17ffffc0001001 3:0 ffff8b23b9d84620 360 lru 00000080:00200000:1.0:1775764496.937695:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000911f982c 360: 1< 0 - - > 2< 1474560 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937696:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000006373799f 00000080:00200000:1.0:1775764496.937697:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 105. 00000080:00200000:1.0:1775764496.937699:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 361 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937723:0:51151:0:(rw26.c:858:ll_write_end()) pos 1478656, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937728:0:51151:0:(rw26.c:916:ll_write_end()) page@000000000f9c7b69[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937730:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000045f1a551 17ffffc0001001 3:0 ffff8b23b9d84000 361 lru 00000080:00200000:1.0:1775764496.937735:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000059359b5e 361: 1< 0 - - > 2< 1478656 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937737:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000000f9c7b69 00000080:00200000:1.0:1775764496.937738:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 106. 00000080:00200000:1.0:1775764496.937740:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 362 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937753:0:51151:0:(rw26.c:858:ll_write_end()) pos 1482752, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937756:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000f33c48e5[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937759:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000003f73815f 17ffffc0001001 3:0 ffff8b23b9d848c0 362 lru 00000080:00200000:1.0:1775764496.937764:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000a1334faa 362: 1< 0 - - > 2< 1482752 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937766:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000f33c48e5 00000080:00200000:1.0:1775764496.937767:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 107. 00000080:00200000:1.0:1775764496.937769:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 363 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937779:0:51151:0:(rw26.c:858:ll_write_end()) pos 1486848, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937782:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000b2084fa4[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937785:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000c9fcfeb4 17ffffc0001001 3:0 ffff8b23b9d842a0 363 lru 00000080:00200000:1.0:1775764496.937791:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000ff02bbd2 363: 1< 0 - - > 2< 1486848 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937793:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000b2084fa4 00000080:00200000:1.0:1775764496.937794:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 108. 00000080:00200000:1.0:1775764496.937796:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 364 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937816:0:51151:0:(rw26.c:858:ll_write_end()) pos 1490944, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937819:0:51151:0:(rw26.c:916:ll_write_end()) page@000000004ef4a766[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937821:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000006088fe74 17ffffc0001001 3:0 ffff8b23b9d84c40 364 lru 00000080:00200000:1.0:1775764496.937825:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000050c07af3 364: 1< 0 - - > 2< 1490944 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937827:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000004ef4a766 00000080:00200000:1.0:1775764496.937828:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 109. 00000080:00200000:1.0:1775764496.937829:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 365 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937841:0:51151:0:(rw26.c:858:ll_write_end()) pos 1495040, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937845:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000091d89904[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937847:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000037dceaa0 17ffffc0001001 3:0 ffff8b23b9d84a80 365 lru 00000080:00200000:1.0:1775764496.937851:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000f6c3fd23 365: 1< 0 - - > 2< 1495040 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937853:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000091d89904 00000080:00200000:1.0:1775764496.937855:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 110. 00000080:00200000:1.0:1775764496.937856:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 366 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937868:0:51151:0:(rw26.c:858:ll_write_end()) pos 1499136, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937872:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000045c10c95[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937875:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000000a84e21c 17ffffc0001001 3:0 ffff8b23b9d84ee0 366 lru 00000080:00200000:1.0:1775764496.937879:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000e2a49a82 366: 1< 0 - - > 2< 1499136 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937881:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000045c10c95 00000080:00200000:1.0:1775764496.937882:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 111. 00000080:00200000:1.0:1775764496.937884:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 367 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937895:0:51151:0:(rw26.c:858:ll_write_end()) pos 1503232, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937899:0:51151:0:(rw26.c:916:ll_write_end()) page@000000005043b525[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937901:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000c3f42cba 17ffffc0001001 3:0 ffff8b23b9d841c0 367 lru 00000080:00200000:1.0:1775764496.937906:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000bcd3c83d 367: 1< 0 - - > 2< 1503232 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937908:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000005043b525 00000080:00200000:1.0:1775764496.937909:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 112. 00000080:00200000:1.0:1775764496.937911:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 368 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937924:0:51151:0:(rw26.c:858:ll_write_end()) pos 1507328, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937927:0:51151:0:(rw26.c:916:ll_write_end()) page@000000002529c9f0[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937929:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000aab3b9f7 17ffffc0001001 3:0 ffff8b23b9d84540 368 lru 00000080:00200000:1.0:1775764496.937934:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000008a713bfb 368: 1< 0 - - > 2< 1507328 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937936:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000002529c9f0 00000080:00200000:1.0:1775764496.937937:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 113. 00000080:00200000:1.0:1775764496.937939:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 369 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937952:0:51151:0:(rw26.c:858:ll_write_end()) pos 1511424, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937955:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000081a57c00[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937958:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000003e12ae73 17ffffc0001001 3:0 ffff8b23b9d84700 369 lru 00000080:00200000:1.0:1775764496.937963:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000000fe31fcb 369: 1< 0 - - > 2< 1511424 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937966:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000081a57c00 00000080:00200000:1.0:1775764496.937967:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 114. 00000080:00200000:1.0:1775764496.937969:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 370 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.937982:0:51151:0:(rw26.c:858:ll_write_end()) pos 1515520, len 4096, copied 4096 00000080:00200000:1.0:1775764496.937986:0:51151:0:(rw26.c:916:ll_write_end()) page@000000008577c393[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.937989:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000b61f6417 17ffffc0001001 3:0 ffff8b23b9d847e0 370 lru 00000080:00200000:1.0:1775764496.937994:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000dc1d2b3a 370: 1< 0 - - > 2< 1515520 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.937996:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000008577c393 00000080:00200000:1.0:1775764496.937997:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 115. 00000080:00200000:1.0:1775764496.937999:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 371 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938012:0:51151:0:(rw26.c:858:ll_write_end()) pos 1519616, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938017:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000f4140219[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938019:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000008ac739f9 17ffffc0001001 3:0 ffff8b23b9d84d20 371 lru 00000080:00200000:1.0:1775764496.938024:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000034a4c984 371: 1< 0 - - > 2< 1519616 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938026:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000f4140219 00000080:00200000:1.0:1775764496.938027:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 116. 00000080:00200000:1.0:1775764496.938029:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 372 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938046:0:51151:0:(rw26.c:858:ll_write_end()) pos 1523712, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938050:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000a61ec66a[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938053:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000c582a338 17ffffc0001021 2:0 ffff8b23b9d84380 372 lru 00000080:00200000:1.0:1775764496.938058:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000f0e0a88b 372: 1< 0 - - > 2< 1523712 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938060:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000a61ec66a 00000080:00200000:1.0:1775764496.938061:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 117. 00000080:00200000:1.0:1775764496.938063:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 373 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938075:0:51151:0:(rw26.c:858:ll_write_end()) pos 1527808, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938079:0:51151:0:(rw26.c:916:ll_write_end()) page@000000007bef0230[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938081:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000014d07ce0 17ffffc0001001 3:0 ffff8b23b9d84460 373 lru 00000080:00200000:1.0:1775764496.938087:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000027179d10 373: 1< 0 - - > 2< 1527808 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938089:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000007bef0230 00000080:00200000:1.0:1775764496.938090:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 118. 00000080:00200000:1.0:1775764496.938092:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 374 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938106:0:51151:0:(rw26.c:858:ll_write_end()) pos 1531904, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938110:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000feda4fb1[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938113:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000007de92480 17ffffc0001001 3:0 ffff8b23b9d840e0 374 lru 00000080:00200000:1.0:1775764496.938119:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000080ee81ea 374: 1< 0 - - > 2< 1531904 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938121:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000feda4fb1 00000080:00200000:1.0:1775764496.938122:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 119. 00000080:00200000:1.0:1775764496.938124:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 375 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938142:0:51151:0:(rw26.c:858:ll_write_end()) pos 1536000, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938145:0:51151:0:(rw26.c:916:ll_write_end()) page@000000005e3c1d77[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938148:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000f208238f 17ffffc0001001 3:0 ffff8b23b9fb57e0 375 lru 00000080:00200000:1.0:1775764496.938154:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000055c0ee77 375: 1< 0 - - > 2< 1536000 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938156:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000005e3c1d77 00000080:00200000:1.0:1775764496.938158:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 120. 00000080:00200000:1.0:1775764496.938160:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 376 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938175:0:51151:0:(rw26.c:858:ll_write_end()) pos 1540096, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938179:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000010f0ed2d[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938181:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000029785a0f 17ffffc0001001 3:0 ffff8b23b9fb5d20 376 lru 00000080:00200000:1.0:1775764496.938186:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000fb83239d 376: 1< 0 - - > 2< 1540096 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938188:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000010f0ed2d 00000080:00200000:1.0:1775764496.938189:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 121. 00000080:00200000:1.0:1775764496.938192:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 377 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938209:0:51151:0:(rw26.c:858:ll_write_end()) pos 1544192, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938212:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000bda7e42e[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938215:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000004bbf95d3 17ffffc0001001 3:0 ffff8b23b9fb51c0 377 lru 00000080:00200000:1.0:1775764496.938220:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000159aab00 377: 1< 0 - - > 2< 1544192 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938222:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000bda7e42e 00000080:00200000:1.0:1775764496.938223:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 122. 00000080:00200000:1.0:1775764496.938225:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 378 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938237:0:51151:0:(rw26.c:858:ll_write_end()) pos 1548288, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938241:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000fe4b5c06[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938243:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000463f7ef1 17ffffc0001001 3:0 ffff8b23b9fb5540 378 lru 00000080:00200000:1.0:1775764496.938249:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000076a497be 378: 1< 0 - - > 2< 1548288 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938252:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000fe4b5c06 00000080:00200000:1.0:1775764496.938253:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 123. 00000080:00200000:1.0:1775764496.938255:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 379 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938268:0:51151:0:(rw26.c:858:ll_write_end()) pos 1552384, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938272:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000019ee7307[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938274:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000004055948b 17ffffc0001001 3:0 ffff8b23b9fb5700 379 lru 00000080:00200000:1.0:1775764496.938278:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000c1d7c6b2 379: 1< 0 - - > 2< 1552384 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938280:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000019ee7307 00000080:00200000:1.0:1775764496.938281:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 124. 00000080:00200000:1.0:1775764496.938283:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 380 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938296:0:51151:0:(rw26.c:858:ll_write_end()) pos 1556480, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938300:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000ce2bc6eb[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938302:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000008e1fa746 17ffffc0001001 3:0 ffff8b23b9fb5460 380 lru 00000080:00200000:1.0:1775764496.938306:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000a4191dfe 380: 1< 0 - - > 2< 1556480 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938309:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000ce2bc6eb 00000080:00200000:1.0:1775764496.938310:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 125. 00000080:00200000:1.0:1775764496.938312:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 381 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938326:0:51151:0:(rw26.c:858:ll_write_end()) pos 1560576, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938329:0:51151:0:(rw26.c:916:ll_write_end()) page@000000006b1682d4[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938332:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000009edb978d 17ffffc0001001 3:0 ffff8b23b9fb50e0 381 lru 00000080:00200000:1.0:1775764496.938336:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000090f0108 381: 1< 0 - - > 2< 1560576 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938339:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000006b1682d4 00000080:00200000:1.0:1775764496.938339:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 126. 00000080:00200000:1.0:1775764496.938341:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 382 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938358:0:51151:0:(rw26.c:858:ll_write_end()) pos 1564672, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938362:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000076640a23[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938364:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000980b7102 17ffffc0001001 3:0 ffff8b23b9fb5ee0 382 lru 00000080:00200000:1.0:1775764496.938369:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000c4d93406 382: 1< 0 - - > 2< 1564672 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938371:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000076640a23 00000080:00200000:1.0:1775764496.938372:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 127. 00000080:00200000:1.0:1775764496.938374:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 383 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938388:0:51151:0:(rw26.c:858:ll_write_end()) pos 1568768, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938391:0:51151:0:(rw26.c:916:ll_write_end()) page@000000002c4cc475[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938394:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000a027bbb4 17ffffc0001001 3:0 ffff8b23b9fb5a80 383 lru 00000080:00200000:1.0:1775764496.938398:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000012854b38 383: 1< 0 - - > 2< 1568768 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938400:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000002c4cc475 00000080:00200000:1.0:1775764496.938401:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 128. 00000080:00200000:1.0:1775764496.938403:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 384 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938424:0:51151:0:(rw26.c:858:ll_write_end()) pos 1572864, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938428:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000820797c4[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938432:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000087ce879e 17ffffc0001001 3:0 ffff8b23b9fb5c40 384 lru 00000080:00200000:1.0:1775764496.938436:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000139bc451 384: 1< 0 - - > 2< 1572864 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938438:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000820797c4 00000080:00200000:1.0:1775764496.938439:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 129. 00000080:00200000:1.0:1775764496.938441:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 385 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938498:0:51151:0:(rw26.c:858:ll_write_end()) pos 1576960, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938502:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000030479255[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938504:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000e983540c 17ffffc0001001 3:0 ffff8b23b9fb5e00 385 lru 00000080:00200000:1.0:1775764496.938510:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000e288b3ad 385: 1< 0 - - > 2< 1576960 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938512:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000030479255 00000080:00200000:1.0:1775764496.938513:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 130. 00000080:00200000:1.0:1775764496.938516:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 386 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938532:0:51151:0:(rw26.c:858:ll_write_end()) pos 1581056, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938535:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000013e0dcf9[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938538:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000561a658a 17ffffc0001001 3:0 ffff8b23b9fb5b60 386 lru 00000080:00200000:1.0:1775764496.938543:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000b6856dec 386: 1< 0 - - > 2< 1581056 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938545:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000013e0dcf9 00000080:00200000:1.0:1775764496.938546:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 131. 00000080:00200000:1.0:1775764496.938548:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 387 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938567:0:51151:0:(rw26.c:858:ll_write_end()) pos 1585152, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938571:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000885e29d7[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938573:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000002c56cbd6 17ffffc0001021 2:0 ffff8b23b9fb59a0 387 lru 00000080:00200000:1.0:1775764496.938578:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000008d889e1b 387: 1< 0 - - > 2< 1585152 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938580:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000885e29d7 00000080:00200000:1.0:1775764496.938581:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 132. 00000080:00200000:1.0:1775764496.938583:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 388 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938598:0:51151:0:(rw26.c:858:ll_write_end()) pos 1589248, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938601:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000f56eb2a8[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938603:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000e0a68511 17ffffc0001001 3:0 ffff8b23b9fb5620 388 lru 00000080:00200000:1.0:1775764496.938608:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000d752ed8e 388: 1< 0 - - > 2< 1589248 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938610:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000f56eb2a8 00000080:00200000:1.0:1775764496.938611:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 133. 00000080:00200000:1.0:1775764496.938612:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 389 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938627:0:51151:0:(rw26.c:858:ll_write_end()) pos 1593344, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938630:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000a159cccb[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938632:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000038423957 17ffffc0001001 3:0 ffff8b23b9fb5000 389 lru 00000080:00200000:1.0:1775764496.938637:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000c88d33c2 389: 1< 0 - - > 2< 1593344 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938638:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000a159cccb 00000080:00200000:1.0:1775764496.938639:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 134. 00000080:00200000:1.0:1775764496.938641:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 390 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938652:0:51151:0:(rw26.c:858:ll_write_end()) pos 1597440, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938655:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000e419bac1[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938657:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000001aa1849a 17ffffc0001001 3:0 ffff8b23b9fb58c0 390 lru 00000080:00200000:1.0:1775764496.938662:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000003529da2d 390: 1< 0 - - > 2< 1597440 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938664:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000e419bac1 00000080:00200000:1.0:1775764496.938665:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 135. 00000080:00200000:1.0:1775764496.938666:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 391 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938677:0:51151:0:(rw26.c:858:ll_write_end()) pos 1601536, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938680:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000f81dfbe7[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938682:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000e34660be 17ffffc0001001 3:0 ffff8b23b9fb52a0 391 lru 00000080:00200000:1.0:1775764496.938687:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000006871cb5f 391: 1< 0 - - > 2< 1601536 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938688:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000f81dfbe7 00000080:00200000:1.0:1775764496.938689:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 136. 00000080:00200000:1.0:1775764496.938691:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 392 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938705:0:51151:0:(rw26.c:858:ll_write_end()) pos 1605632, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938720:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000438ad4e6[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938723:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000230bd681 17ffffc0001001 3:0 ffff8b23bae27000 392 lru 00000080:00200000:1.0:1775764496.938728:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000d8140102 392: 1< 0 - - > 2< 1605632 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938729:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000438ad4e6 00000080:00200000:1.0:1775764496.938730:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 137. 00000080:00200000:1.0:1775764496.938733:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 393 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938746:0:51151:0:(rw26.c:858:ll_write_end()) pos 1609728, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938749:0:51151:0:(rw26.c:916:ll_write_end()) page@000000008c8905d0[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938751:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000b2673ba8 17ffffc0001001 3:0 ffff8b23bae27620 393 lru 00000080:00200000:1.0:1775764496.938755:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000885c40e0 393: 1< 0 - - > 2< 1609728 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938757:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000008c8905d0 00000080:00200000:1.0:1775764496.938759:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 138. 00000080:00200000:1.0:1775764496.938760:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 394 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938771:0:51151:0:(rw26.c:858:ll_write_end()) pos 1613824, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938774:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000c5421520[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938777:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000004eaffe0f 17ffffc0001001 3:0 ffff8b23bae279a0 394 lru 00000080:00200000:1.0:1775764496.938782:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000068076fd4 394: 1< 0 - - > 2< 1613824 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938784:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000c5421520 00000080:00200000:1.0:1775764496.938785:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 139. 00000080:00200000:1.0:1775764496.938788:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 395 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938799:0:51151:0:(rw26.c:858:ll_write_end()) pos 1617920, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938803:0:51151:0:(rw26.c:916:ll_write_end()) page@000000006b8f96af[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938806:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000007dbf95f9 17ffffc0001001 3:0 ffff8b23bae27b60 395 lru 00000080:00200000:1.0:1775764496.938810:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000036e5c50b 395: 1< 0 - - > 2< 1617920 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938813:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000006b8f96af 00000080:00200000:1.0:1775764496.938814:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 140. 00000080:00200000:1.0:1775764496.938816:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 396 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938830:0:51151:0:(rw26.c:858:ll_write_end()) pos 1622016, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938834:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000bd17fdf2[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938836:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000030e86fb1 17ffffc0001001 3:0 ffff8b23bae27e00 396 lru 00000080:00200000:1.0:1775764496.938841:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000007ff9a2ce 396: 1< 0 - - > 2< 1622016 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938843:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000bd17fdf2 00000080:00200000:1.0:1775764496.938844:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 141. 00000080:00200000:1.0:1775764496.938846:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 397 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938859:0:51151:0:(rw26.c:858:ll_write_end()) pos 1626112, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938863:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000b71996d8[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938865:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000009da485a4 17ffffc0001001 3:0 ffff8b23bae27c40 397 lru 00000080:00200000:1.0:1775764496.938871:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000eba44723 397: 1< 0 - - > 2< 1626112 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938873:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000b71996d8 00000080:00200000:1.0:1775764496.938874:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 142. 00000080:00200000:1.0:1775764496.938876:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 398 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938889:0:51151:0:(rw26.c:858:ll_write_end()) pos 1630208, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938893:0:51151:0:(rw26.c:916:ll_write_end()) page@000000008326d92c[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938895:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000feeeebaa 17ffffc0001001 3:0 ffff8b23bae27a80 398 lru 00000080:00200000:1.0:1775764496.938899:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000584cc792 398: 1< 0 - - > 2< 1630208 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938900:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000008326d92c 00000080:00200000:1.0:1775764496.938901:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 143. 00000080:00200000:1.0:1775764496.938903:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 399 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938917:0:51151:0:(rw26.c:858:ll_write_end()) pos 1634304, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938920:0:51151:0:(rw26.c:916:ll_write_end()) page@000000000bc7b651[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938923:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000d073d2be 17ffffc0001001 3:0 ffff8b23bae27ee0 399 lru 00000080:00200000:1.0:1775764496.938928:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000075b49b8d 399: 1< 0 - - > 2< 1634304 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938930:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000000bc7b651 00000080:00200000:1.0:1775764496.938931:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 144. 00000080:00200000:1.0:1775764496.938933:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 400 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938951:0:51151:0:(rw26.c:858:ll_write_end()) pos 1638400, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938955:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000f4dcb409[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938957:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000072cee928 17ffffc0001001 3:0 ffff8b23bae271c0 400 lru 00000080:00200000:1.0:1775764496.938962:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000001c1c988b 400: 1< 0 - - > 2< 1638400 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938963:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000f4dcb409 00000080:00200000:1.0:1775764496.938964:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 145. 00000080:00200000:1.0:1775764496.938966:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 401 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.938980:0:51151:0:(rw26.c:858:ll_write_end()) pos 1642496, len 4096, copied 4096 00000080:00200000:1.0:1775764496.938984:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000039e2f935[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.938986:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000af2cce83 17ffffc0001001 3:0 ffff8b23bae278c0 401 lru 00000080:00200000:1.0:1775764496.938990:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000043dcc352 401: 1< 0 - - > 2< 1642496 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.938992:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000039e2f935 00000080:00200000:1.0:1775764496.938993:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 146. 00000080:00200000:1.0:1775764496.938995:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 402 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939013:0:51151:0:(rw26.c:858:ll_write_end()) pos 1646592, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939017:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000dc8791fd[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939020:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000002b0904a 17ffffc0001021 2:0 ffff8b23bae272a0 402 lru 00000080:00200000:1.0:1775764496.939025:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000dd4eb9c1 402: 1< 0 - - > 2< 1646592 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939027:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000dc8791fd 00000080:00200000:1.0:1775764496.939028:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 147. 00000080:00200000:1.0:1775764496.939030:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 403 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939045:0:51151:0:(rw26.c:858:ll_write_end()) pos 1650688, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939048:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000275606c9[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939051:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000ad9ad51c 17ffffc0001001 3:0 ffff8b23bae270e0 403 lru 00000080:00200000:1.0:1775764496.939056:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000019539f33 403: 1< 0 - - > 2< 1650688 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939058:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000275606c9 00000080:00200000:1.0:1775764496.939059:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 148. 00000080:00200000:1.0:1775764496.939061:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 404 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939073:0:51151:0:(rw26.c:858:ll_write_end()) pos 1654784, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939077:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000ed1c8ef4[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939079:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000059a56619 17ffffc0001001 3:0 ffff8b23bae27460 404 lru 00000080:00200000:1.0:1775764496.939084:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000004f66ca88 404: 1< 0 - - > 2< 1654784 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939086:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000ed1c8ef4 00000080:00200000:1.0:1775764496.939087:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 149. 00000080:00200000:1.0:1775764496.939089:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 405 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939101:0:51151:0:(rw26.c:858:ll_write_end()) pos 1658880, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939104:0:51151:0:(rw26.c:916:ll_write_end()) page@000000005d3de1fc[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939107:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000009795afae 17ffffc0001001 3:0 ffff8b23bae27380 405 lru 00000080:00200000:1.0:1775764496.939112:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000038c4cccd 405: 1< 0 - - > 2< 1658880 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939114:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000005d3de1fc 00000080:00200000:1.0:1775764496.939115:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 150. 00000080:00200000:1.0:1775764496.939117:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 406 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939131:0:51151:0:(rw26.c:858:ll_write_end()) pos 1662976, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939135:0:51151:0:(rw26.c:916:ll_write_end()) page@000000001726896d[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939137:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000aab34614 17ffffc0001001 3:0 ffff8b23bae27d20 406 lru 00000080:00200000:1.0:1775764496.939141:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000aec71233 406: 1< 0 - - > 2< 1662976 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939143:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000001726896d 00000080:00200000:1.0:1775764496.939144:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 151. 00000080:00200000:1.0:1775764496.939146:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 407 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939160:0:51151:0:(rw26.c:858:ll_write_end()) pos 1667072, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939163:0:51151:0:(rw26.c:916:ll_write_end()) page@000000002453c546[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939165:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000c170ad6d 17ffffc0001001 3:0 ffff8b23bae277e0 407 lru 00000080:00200000:1.0:1775764496.939170:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000d4d3bab2 407: 1< 0 - - > 2< 1667072 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939172:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000002453c546 00000080:00200000:1.0:1775764496.939172:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 152. 00000080:00200000:1.0:1775764496.939174:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 408 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939186:0:51151:0:(rw26.c:858:ll_write_end()) pos 1671168, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939189:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000040bdcb58[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939191:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000007ac0e8eb 17ffffc0001001 3:0 ffff8b23bae27700 408 lru 00000080:00200000:1.0:1775764496.939196:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000000f07e7c4 408: 1< 0 - - > 2< 1671168 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939198:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000040bdcb58 00000080:00200000:1.0:1775764496.939199:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 153. 00000080:00200000:1.0:1775764496.939201:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 409 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939215:0:51151:0:(rw26.c:858:ll_write_end()) pos 1675264, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939219:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000bf0c49ee[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939221:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000f08a2633 17ffffc0001001 3:0 ffff8b23b9d1a2a0 409 lru 00000080:00200000:1.0:1775764496.939227:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000008ce647c7 409: 1< 0 - - > 2< 1675264 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939229:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000bf0c49ee 00000080:00200000:1.0:1775764496.939230:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 154. 00000080:00200000:1.0:1775764496.939232:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 410 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939249:0:51151:0:(rw26.c:858:ll_write_end()) pos 1679360, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939252:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000ef03a14d[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939255:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000a147e252 17ffffc0001001 3:0 ffff8b23b9d1a8c0 410 lru 00000080:00200000:1.0:1775764496.939259:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000006e2f15de 410: 1< 0 - - > 2< 1679360 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939261:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000ef03a14d 00000080:00200000:1.0:1775764496.939262:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 155. 00000080:00200000:1.0:1775764496.939264:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 411 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939276:0:51151:0:(rw26.c:858:ll_write_end()) pos 1683456, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939280:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000004e6431c[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939282:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000b95b6088 17ffffc0001001 3:0 ffff8b23b9d1a000 411 lru 00000080:00200000:1.0:1775764496.939287:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000002d6dc505 411: 1< 0 - - > 2< 1683456 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939289:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000004e6431c 00000080:00200000:1.0:1775764496.939290:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 156. 00000080:00200000:1.0:1775764496.939291:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 412 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939303:0:51151:0:(rw26.c:858:ll_write_end()) pos 1687552, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939306:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000c432adad[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939309:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000000ee26792 17ffffc0001001 3:0 ffff8b23b9d1a620 412 lru 00000080:00200000:1.0:1775764496.939314:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000009b9039f4 412: 1< 0 - - > 2< 1687552 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939317:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000c432adad 00000080:00200000:1.0:1775764496.939318:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 157. 00000080:00200000:1.0:1775764496.939321:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 413 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939334:0:51151:0:(rw26.c:858:ll_write_end()) pos 1691648, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939338:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000071c1c945[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939340:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000a6b153de 17ffffc0001001 3:0 ffff8b23b9d1a460 413 lru 00000080:00200000:1.0:1775764496.939345:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000003389e803 413: 1< 0 - - > 2< 1691648 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939348:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000071c1c945 00000080:00200000:1.0:1775764496.939348:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 158. 00000080:00200000:1.0:1775764496.939350:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 414 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939364:0:51151:0:(rw26.c:858:ll_write_end()) pos 1695744, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939367:0:51151:0:(rw26.c:916:ll_write_end()) page@000000005f15976c[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939370:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000d876d926 17ffffc0001001 3:0 ffff8b23b9d1a0e0 414 lru 00000080:00200000:1.0:1775764496.939375:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000049691ae5 414: 1< 0 - - > 2< 1695744 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939377:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000005f15976c 00000080:00200000:1.0:1775764496.939378:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 159. 00000080:00200000:1.0:1775764496.939380:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 415 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939392:0:51151:0:(rw26.c:858:ll_write_end()) pos 1699840, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939396:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000ca995bc9[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939398:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000e7797884 17ffffc0001001 3:0 ffff8b23b9d1a380 415 lru 00000080:00200000:1.0:1775764496.939403:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000005c46786 415: 1< 0 - - > 2< 1699840 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939405:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000ca995bc9 00000080:00200000:1.0:1775764496.939406:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 160. 00000080:00200000:1.0:1775764496.939407:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 416 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939422:0:51151:0:(rw26.c:858:ll_write_end()) pos 1703936, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939426:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000d5b63055[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939429:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000093a28092 17ffffc0001001 3:0 ffff8b23b9d1ad20 416 lru 00000080:00200000:1.0:1775764496.939434:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000008ba825fe 416: 1< 0 - - > 2< 1703936 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939436:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000d5b63055 00000080:00200000:1.0:1775764496.939437:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 161. 00000080:00200000:1.0:1775764496.939439:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 417 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939487:0:51151:0:(rw26.c:858:ll_write_end()) pos 1708032, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939492:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000093c12b06[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939494:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000094b92e12 17ffffc0001021 2:0 ffff8b23b9d1a7e0 417 lru 00000080:00200000:1.0:1775764496.939500:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000004b87e86e 417: 1< 0 - - > 2< 1708032 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939502:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000093c12b06 00000080:00200000:1.0:1775764496.939503:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 162. 00000080:00200000:1.0:1775764496.939505:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 418 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939520:0:51151:0:(rw26.c:858:ll_write_end()) pos 1712128, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939524:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000055903231[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939526:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000007d8d22a2 17ffffc0001001 3:0 ffff8b23b9d1a700 418 lru 00000080:00200000:1.0:1775764496.939531:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000008feba108 418: 1< 0 - - > 2< 1712128 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939533:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000055903231 00000080:00200000:1.0:1775764496.939534:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 163. 00000080:00200000:1.0:1775764496.939536:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 419 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939547:0:51151:0:(rw26.c:858:ll_write_end()) pos 1716224, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939551:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000657c08dd[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939553:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000003473bf45 17ffffc0001001 3:0 ffff8b23b9d1a540 419 lru 00000080:00200000:1.0:1775764496.939558:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000c64d7aa1 419: 1< 0 - - > 2< 1716224 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939560:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000657c08dd 00000080:00200000:1.0:1775764496.939561:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 164. 00000080:00200000:1.0:1775764496.939562:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 420 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939574:0:51151:0:(rw26.c:858:ll_write_end()) pos 1720320, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939577:0:51151:0:(rw26.c:916:ll_write_end()) page@000000006bc1516f[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939580:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000003e01f583 17ffffc0001001 3:0 ffff8b23b9d1a1c0 420 lru 00000080:00200000:1.0:1775764496.939585:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000004a3d1dec 420: 1< 0 - - > 2< 1720320 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939586:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000006bc1516f 00000080:00200000:1.0:1775764496.939587:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 165. 00000080:00200000:1.0:1775764496.939589:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 421 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939601:0:51151:0:(rw26.c:858:ll_write_end()) pos 1724416, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939605:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000d8e203de[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939607:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000006cb511cd 17ffffc0001001 3:0 ffff8b23b9d1aee0 421 lru 00000080:00200000:1.0:1775764496.939611:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000002df45200 421: 1< 0 - - > 2< 1724416 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939613:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000d8e203de 00000080:00200000:1.0:1775764496.939614:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 166. 00000080:00200000:1.0:1775764496.939616:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 422 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939627:0:51151:0:(rw26.c:858:ll_write_end()) pos 1728512, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939630:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000092e87784[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939632:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000c764f2ba 17ffffc0001001 3:0 ffff8b23b9d1aa80 422 lru 00000080:00200000:1.0:1775764496.939637:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000b061d702 422: 1< 0 - - > 2< 1728512 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939638:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000092e87784 00000080:00200000:1.0:1775764496.939639:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 167. 00000080:00200000:1.0:1775764496.939641:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 423 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939652:0:51151:0:(rw26.c:858:ll_write_end()) pos 1732608, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939656:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000e0831a21[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939658:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000000367e8e9 17ffffc0001001 3:0 ffff8b23b9d1ac40 423 lru 00000080:00200000:1.0:1775764496.939664:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000540e0896 423: 1< 0 - - > 2< 1732608 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939666:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000e0831a21 00000080:00200000:1.0:1775764496.939666:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 168. 00000080:00200000:1.0:1775764496.939668:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 424 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939684:0:51151:0:(rw26.c:858:ll_write_end()) pos 1736704, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939689:0:51151:0:(rw26.c:916:ll_write_end()) page@000000006119448a[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939692:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000006fa5bf05 17ffffc0001001 3:0 ffff8b23b9d1ae00 424 lru 00000080:00200000:1.0:1775764496.939698:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000e742b31c 424: 1< 0 - - > 2< 1736704 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939700:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000006119448a 00000080:00200000:1.0:1775764496.939701:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 169. 00000080:00200000:1.0:1775764496.939703:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 425 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939732:0:51151:0:(rw26.c:858:ll_write_end()) pos 1740800, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939737:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000da7cc16c[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939740:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000005f4563ff 17ffffc0001001 3:0 ffff8b23b9d1ab60 425 lru 00000080:00200000:1.0:1775764496.939745:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000090569539 425: 1< 0 - - > 2< 1740800 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939747:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000da7cc16c 00000080:00200000:1.0:1775764496.939748:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 170. 00000080:00200000:1.0:1775764496.939750:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 426 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939772:0:51151:0:(rw26.c:858:ll_write_end()) pos 1744896, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939776:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000075b1634c[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939778:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000454862e5 17ffffc0001001 3:0 ffff8b23bae430e0 426 lru 00000080:00200000:1.0:1775764496.939783:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000099cb1948 426: 1< 0 - - > 2< 1744896 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939785:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000075b1634c 00000080:00200000:1.0:1775764496.939786:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 171. 00000080:00200000:1.0:1775764496.939787:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 427 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939802:0:51151:0:(rw26.c:858:ll_write_end()) pos 1748992, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939806:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000da000975[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939808:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000085ba4fb6 17ffffc0001001 3:0 ffff8b23bae432a0 427 lru 00000080:00200000:1.0:1775764496.939814:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000eeb30228 427: 1< 0 - - > 2< 1748992 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939815:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000da000975 00000080:00200000:1.0:1775764496.939816:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 172. 00000080:00200000:1.0:1775764496.939818:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 428 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939832:0:51151:0:(rw26.c:858:ll_write_end()) pos 1753088, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939836:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000e62a8af3[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939839:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000fd88102b 17ffffc0001001 3:0 ffff8b23bae438c0 428 lru 00000080:00200000:1.0:1775764496.939844:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000028e492d 428: 1< 0 - - > 2< 1753088 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939846:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000e62a8af3 00000080:00200000:1.0:1775764496.939847:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 173. 00000080:00200000:1.0:1775764496.939849:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 429 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939864:0:51151:0:(rw26.c:858:ll_write_end()) pos 1757184, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939867:0:51151:0:(rw26.c:916:ll_write_end()) page@000000005f8ec195[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939870:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000a1c3e043 17ffffc0001001 3:0 ffff8b23bae43000 429 lru 00000080:00200000:1.0:1775764496.939874:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000f71e04a0 429: 1< 0 - - > 2< 1757184 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939876:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000005f8ec195 00000080:00200000:1.0:1775764496.939877:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 174. 00000080:00200000:1.0:1775764496.939878:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 430 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939891:0:51151:0:(rw26.c:858:ll_write_end()) pos 1761280, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939895:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000099c54279[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939897:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000053b49ac3 17ffffc0001001 3:0 ffff8b23bae43620 430 lru 00000080:00200000:1.0:1775764496.939901:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000000ac8c070 430: 1< 0 - - > 2< 1761280 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939903:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000099c54279 00000080:00200000:1.0:1775764496.939904:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 175. 00000080:00200000:1.0:1775764496.939906:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 431 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939922:0:51151:0:(rw26.c:858:ll_write_end()) pos 1765376, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939926:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000bb0d2e06[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939928:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000b6ba4932 17ffffc0001001 3:0 ffff8b23bae439a0 431 lru 00000080:00200000:1.0:1775764496.939933:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000002b7d5099 431: 1< 0 - - > 2< 1765376 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939934:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000bb0d2e06 00000080:00200000:1.0:1775764496.939935:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 176. 00000080:00200000:1.0:1775764496.939937:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 432 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939955:0:51151:0:(rw26.c:858:ll_write_end()) pos 1769472, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939959:0:51151:0:(rw26.c:916:ll_write_end()) page@000000009b100ca5[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939961:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000cac2079c 17ffffc0001021 2:0 ffff8b23bae43b60 432 lru 00000080:00200000:1.0:1775764496.939966:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000bcf2a5ac 432: 1< 0 - - > 2< 1769472 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939968:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000009b100ca5 00000080:00200000:1.0:1775764496.939969:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 177. 00000080:00200000:1.0:1775764496.939971:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 433 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.939983:0:51151:0:(rw26.c:858:ll_write_end()) pos 1773568, len 4096, copied 4096 00000080:00200000:1.0:1775764496.939987:0:51151:0:(rw26.c:916:ll_write_end()) page@000000007f357af1[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.939990:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000014e8eb14 17ffffc0001001 3:0 ffff8b23bae43e00 433 lru 00000080:00200000:1.0:1775764496.939995:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000007f0f6c2a 433: 1< 0 - - > 2< 1773568 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.939997:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000007f357af1 00000080:00200000:1.0:1775764496.939998:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 178. 00000080:00200000:1.0:1775764496.940000:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 434 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.940013:0:51151:0:(rw26.c:858:ll_write_end()) pos 1777664, len 4096, copied 4096 00000080:00200000:1.0:1775764496.940016:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000078baf3e1[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.940018:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000e4dabacd 17ffffc0001001 3:0 ffff8b23bae43c40 434 lru 00000080:00200000:1.0:1775764496.940023:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000065dc78b2 434: 1< 0 - - > 2< 1777664 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.940025:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000078baf3e1 00000080:00200000:1.0:1775764496.940026:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 179. 00000080:00200000:1.0:1775764496.940028:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 435 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.940038:0:51151:0:(rw26.c:858:ll_write_end()) pos 1781760, len 4096, copied 4096 00000080:00200000:1.0:1775764496.940041:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000817af161[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.940043:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000f473763e 17ffffc0001001 3:0 ffff8b23bae43a80 435 lru 00000080:00200000:1.0:1775764496.940048:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000736b7a8f 435: 1< 0 - - > 2< 1781760 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.940050:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000817af161 00000080:00200000:1.0:1775764496.940051:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 180. 00000080:00200000:1.0:1775764496.940053:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 436 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.940064:0:51151:0:(rw26.c:858:ll_write_end()) pos 1785856, len 4096, copied 4096 00000080:00200000:1.0:1775764496.940067:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000e899a57d[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.940070:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000002e8e983e 17ffffc0001001 3:0 ffff8b23bae43ee0 436 lru 00000080:00200000:1.0:1775764496.940075:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000005984fab0 436: 1< 0 - - > 2< 1785856 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.940077:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000e899a57d 00000080:00200000:1.0:1775764496.940078:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 181. 00000080:00200000:1.0:1775764496.940079:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 437 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.940091:0:51151:0:(rw26.c:858:ll_write_end()) pos 1789952, len 4096, copied 4096 00000080:00200000:1.0:1775764496.940095:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000f14116d8[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.940097:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000bffccf6e 17ffffc0001001 3:0 ffff8b23bae431c0 437 lru 00000080:00200000:1.0:1775764496.940102:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000008f9fa17b 437: 1< 0 - - > 2< 1789952 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.940104:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000f14116d8 00000080:00200000:1.0:1775764496.940105:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 182. 00000080:00200000:1.0:1775764496.940107:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 438 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.940119:0:51151:0:(rw26.c:858:ll_write_end()) pos 1794048, len 4096, copied 4096 00000080:00200000:1.0:1775764496.940122:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000275e233d[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.940124:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000c9e06995 17ffffc0001001 3:0 ffff8b23bae43540 438 lru 00000080:00200000:1.0:1775764496.940129:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000004f394412 438: 1< 0 - - > 2< 1794048 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.940131:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000275e233d 00000080:00200000:1.0:1775764496.940132:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 183. 00000080:00200000:1.0:1775764496.940133:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 439 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.940144:0:51151:0:(rw26.c:858:ll_write_end()) pos 1798144, len 4096, copied 4096 00000080:00200000:1.0:1775764496.953246:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000004db398f[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.953254:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000007b3eeda 17ffffc0001001 3:0 ffff8b23bae43700 439 lru 00000080:00200000:1.0:1775764496.953263:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000ed959f7c 439: 1< 0 - - > 2< 1798144 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.953265:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000004db398f 00000080:00200000:1.0:1775764496.953267:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 184. 00000080:00200000:1.0:1775764496.953274:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 440 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.953329:0:51151:0:(rw26.c:858:ll_write_end()) pos 1802240, len 4096, copied 4096 00000080:00200000:1.0:1775764496.953334:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000295b317d[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.953337:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000001c4f229a 17ffffc0001001 3:0 ffff8b23bae437e0 440 lru 00000080:00200000:1.0:1775764496.953342:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000abcf97e5 440: 1< 0 - - > 2< 1802240 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.953344:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000295b317d 00000080:00200000:1.0:1775764496.953345:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 185. 00000080:00200000:1.0:1775764496.953347:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 441 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.953361:0:51151:0:(rw26.c:858:ll_write_end()) pos 1806336, len 4096, copied 4096 00000080:00200000:1.0:1775764496.953364:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000050c7a7d5[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.953366:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000015e15232 17ffffc0001001 3:0 ffff8b23bae43d20 441 lru 00000080:00200000:1.0:1775764496.953371:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000011ffd971 441: 1< 0 - - > 2< 1806336 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.953373:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000050c7a7d5 00000080:00200000:1.0:1775764496.953374:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 186. 00000080:00200000:1.0:1775764496.953376:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 442 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.953388:0:51151:0:(rw26.c:858:ll_write_end()) pos 1810432, len 4096, copied 4096 00000080:00200000:1.0:1775764496.953403:0:51151:0:(rw26.c:916:ll_write_end()) page@0000000028314a56[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.953406:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000025ee1c8c 17ffffc0001001 3:0 ffff8b23bae43380 442 lru 00000080:00200000:1.0:1775764496.953411:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@0000000004a8f5be 442: 1< 0 - - > 2< 1810432 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.953413:0:51151:0:(rw26.c:916:ll_write_end()) end page@0000000028314a56 00000080:00200000:1.0:1775764496.953414:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 187. 00000080:00200000:1.0:1775764496.953416:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 443 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.953432:0:51151:0:(rw26.c:858:ll_write_end()) pos 1814528, len 4096, copied 4096 00000080:00200000:1.0:1775764496.953436:0:51151:0:(rw26.c:916:ll_write_end()) page@00000000a05c9e21[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.953438:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000007258f662 17ffffc0001001 3:0 ffff8b23bae43460 443 lru 00000080:00200000:1.0:1775764496.953443:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000006dc770a9 443: 1< 0 - - > 2< 1814528 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.953445:0:51151:0:(rw26.c:916:ll_write_end()) end page@00000000a05c9e21 00000080:00200000:1.0:1775764496.953446:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 188. 00000080:00200000:1.0:1775764496.953448:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 444 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.953491:0:51151:0:(rw26.c:858:ll_write_end()) pos 1818624, len 4096, copied 4096 00000080:00200000:1.0:1775764496.953495:0:51151:0:(rw26.c:916:ll_write_end()) page@000000009fdaa990[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.953498:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000c8b8fd1b 17ffffc0001001 3:0 ffff8b238a05fc40 444 lru 00000080:00200000:1.0:1775764496.953503:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000aa875dd8 444: 1< 0 - - > 2< 1818624 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.953506:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000009fdaa990 00000080:00200000:1.0:1775764496.953507:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 189. 00000080:00200000:1.0:1775764496.953509:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 445 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.953525:0:51151:0:(rw26.c:858:ll_write_end()) pos 1822720, len 4096, copied 4096 00000080:00200000:1.0:1775764496.953529:0:51151:0:(rw26.c:916:ll_write_end()) page@000000009f509e0b[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.953532:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @0000000047026c8a 17ffffc0001001 3:0 ffff8b238a05fa80 445 lru 00000080:00200000:1.0:1775764496.953537:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@00000000b8e3863b 445: 1< 0 - - > 2< 1822720 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.953539:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000009f509e0b 00000080:00200000:1.0:1775764496.953540:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 190. 00000080:00200000:1.0:1775764496.953542:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 446 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.953555:0:51151:0:(rw26.c:858:ll_write_end()) pos 1826816, len 4096, copied 4096 00000080:00200000:1.0:1775764496.953559:0:51151:0:(rw26.c:916:ll_write_end()) page@000000005f78ceec[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.953561:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @000000002ef6a0c6 17ffffc0001001 3:0 ffff8b238a05fee0 446 lru 00000080:00200000:1.0:1775764496.953566:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000002af62ca0 446: 1< 0 - - > 2< 1826816 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.953568:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000005f78ceec 00000080:00200000:1.0:1775764496.953569:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 191. 00000080:00200000:1.0:1775764496.953570:0:51151:0:(rw26.c:663:ll_write_begin()) Writing 447 of 0 to 4096 bytes 00000080:00200000:1.0:1775764496.953590:0:51151:0:(rw26.c:858:ll_write_end()) pos 1830912, len 4096, copied 4096 00000080:00200000:1.0:1775764496.953606:0:51151:0:(rw26.c:916:ll_write_end()) page@000000005d2e0889[3 00000000a0c51334 2 1 000000005399467a] 00000080:00200000:1.0:1775764496.954093:0:51151:0:(rw26.c:916:ll_write_end()) vmpage @00000000a3ec6abf 17ffffc0001021 2:0 ffff8b238a05f1c0 447 lru 00000080:00200000:1.0:1775764496.954102:0:51151:0:(rw26.c:916:ll_write_end()) osc-page@000000009e3cc954 447: 1< 0 - - > 2< 1830912 0 0 0x0 0x0 | 00000000bf12bc0b 00000000d3d618cf > 3< 0 0 > 4< 0 1 8 1703936 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764496.954104:0:51151:0:(rw26.c:916:ll_write_end()) end page@000000005d2e0889 00000080:00200000:1.0:1775764496.954105:0:51151:0:(rw26.c:916:ll_write_end()) queued cl_page: 192. 00000100:00100000:1.0:1775764496.960513:0:2376:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b2384b87800: committing for last_committed 34359738394 gen 1 00000100:00100000:1.0:1775764496.986794:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098369920:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764497.082486:0:51151:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51151:1862022098370560:192.168.202.128@tcp:44:lhsmtool_posix.0 00000100:00100000:1.0:1775764497.082582:0:51151:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6f3c0 going to sleep for 11 seconds 00000080:00200000:1.0:1775764497.221677:0:51151:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x1f:0x0] updating i_size 4194304 i_blocks 8199 00000080:00200000:1.0:1775764497.221721:0:51151:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x1f:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:1.0:1775764497.221758:0:51151:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab1:0x1f:0x0](ffff8b2390ec0908) cmd=c03066b0 arg=7f262d3efac0 00000080:00200008:1.0:1775764497.221775:0:51151:0:(llite_lib.c:2274:ll_setattr_raw()) START file . :VOLATILE:0000:255BDEBD:fd=09:[0x200003ab1:0x1f:0x0](ffff8b2390ec0908) current size 4194304, valid attrs 1f0, mode 0, uid 0, gid 0, new size 0, atime 1775764495.000000000, mtime 1775764496.000000000, ctime 1775764496.000000000 00010000:00010000:1.0:1775764497.221817:0:51151:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9cea600/0x73cfd86cfc17b152 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x1f:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e0304f433 expref: -99 pid: 51151 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764497.221848:0:51151:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9ceb400/0x73cfd86cfc17b14b lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304f41e expref: -99 pid: 51151 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764497.221994:0:51151:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9cea600/0x73cfd86cfc17b152 lrc: 1/0,0 mode: --/PR res: [0x200003ab1:0x1f:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e0304f433 expref: -99 pid: 51151 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764497.222002:0:51151:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764497.222005:0:51151:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9cea600/0x73cfd86cfc17b152 lrc: 0/0,0 mode: --/PR res: [0x200003ab1:0x1f:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e0304f433 expref: -99 pid: 51151 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000020:00100000:1.0:1775764497.222024:0:51151:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1775764497.222032:0:51151:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd880 x1862022098370944/t0(0) o36->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764497.222056:0:51151:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51151:1862022098370944:192.168.202.128@tcp:36:lhsmtool_posix.0 00000100:00100000:1.0:1775764497.222296:0:51151:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6f480 going to sleep for 11 seconds 00000020:00010000:1.0:1775764497.231184:0:51151:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b3f8 (ffff8b23812ec908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764497.231189:0:51151:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b3f8 (ffff8b23812ec908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764497.231194:0:51151:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24730 (ffff8b23812ec368/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764497.231203:0:51151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ca4d19b/0x73cfd86cfc17b160 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e0304f441 expref: -99 pid: 51151 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764497.231231:0:51151:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ca4d19b/0x73cfd86cfc17b160 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e0304f441 expref: -99 pid: 51151 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000020:00010000:1.0:1775764497.231243:0:51151:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b3f8 (ffff8b23812ec908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764497.231245:0:51151:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24730 (ffff8b23812ec908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764497.231280:0:51151:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x1f:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00800000:00000080:1.0:1775764497.231301:0:51151:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=401866d4 len=384 karg=ffff8b2389501400 uarg=0000000000000000 00000002:00000080:1.0:1775764497.231310:0:51151:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2384b87800: cmd=401866d4 len=384 karg=ffff8b2389501400 uarg=0000000000000000 00000020:00100000:1.0:1775764497.231338:0:51151:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 55, max 7 00000100:00100000:1.0:1775764497.231344:0:51151:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fc380 x1862022098371328/t0(0) o55->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 472/224 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764497.231364:0:51151:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51151:1862022098371328:192.168.202.128@tcp:55:lhsmtool_posix.0 00000100:00100000:1.0:1775764497.232507:0:51151:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6fb40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764497.236860:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764497.247263:0:51151:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51151:1862022098371328:192.168.202.128@tcp:55:lhsmtool_posix.0 00000080:00200008:1.0:1775764497.247303:0:51151:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file . :VOLATILE:0000:255BDEBD:fd=09:[0x200003ab1:0x1f:0x0](ffff8b2390ec0908) current size 4194304, valid attrs 1f0, mode 0, uid 0, gid 0, new size 0, atime 1775764495.000000000, mtime 1775764496.000000000, ctime 1775764496.000000000, rc 0 00000080:00200080:1.0:1775764497.247325:0:51151:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=400866d7 arg=7f2628000b78 00000080:00200000:1.0:1775764497.247386:0:51151:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x1f:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764497.247400:0:51151:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:1.0:1775764497.250823:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118975872 00000100:00100000:1.0:1775764497.250835:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118975872 00000100:00100000:1.0:1775764497.250906:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 83 00000100:00100000:1.0:1775764497.250918:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42946:x1862022118975872:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764497.250937:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ca4d19b/0x73cfd86cfc17b160 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800420000020000 nid: local remote: 0x57b8ff0e0304f441 expref: -99 pid: 51151 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764497.251082:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764497.251090:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42946:x1862022118975872:12345-192.168.202.128@tcp:104: Request processed in 175us (535us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764497.251097:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 83 00800000:00000080:1.0:1775764497.272537:0:51151:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=401866d8 len=64 karg=ffffa60b01d9bcf0 uarg=0000000000000000 00000002:00000080:1.0:1775764497.272549:0:51151:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2384b87800: cmd=401866d8 len=64 karg=ffffa60b01d9bcf0 uarg=0000000000000000 00000020:00100000:1.0:1775764497.272578:0:51151:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 57, max 7 00000100:00100000:1.0:1775764497.272587:0:51151:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fc000 x1862022098371968/t0(0) o57->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764497.272894:0:51151:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51151:1862022098371968:192.168.202.128@tcp:57:lhsmtool_posix.0 00000100:00100000:1.0:1775764497.272974:0:51151:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6fd80 going to sleep for 11 seconds 00000100:00100000:1.0:1775764497.278803:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764497.286567:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:1.0:1775764497.400589:0:51153:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23bacb4400/0x73cfd86cfc17b175 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 51153 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.400601:0:51153:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23bacb4400/0x73cfd86cfc17b175 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 51153 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764497.400610:0:51153:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b238a3bf000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764497.400619:0:51153:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd500 x1862022098373760/t0(0) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 784/131984 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764497.400630:0:51153:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23bacb4400/0x73cfd86cfc17b175 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 51153 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764497.400646:0:51153:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job md5sum:82781e18-e0e3-4f9f-9a2b-041bf0344414:51153:1862022098373760:192.168.202.128@tcp:101:md5sum.0 00000100:00100000:1.0:1775764497.400858:0:51153:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6f840 going to sleep for 11 seconds 00000020:00010000:1.0:1775764497.437964:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b7b8 (ffff8b23812ec128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764497.437969:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b7b8 (ffff8b23812ec128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764497.437974:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e248b0 (ffff8b23812ec6c8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764497.437986:0:51153:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00010000:00010000:1.0:1775764497.438002:0:51153:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000020:00010000:1.0:1775764497.438019:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b7b8 (ffff8b23812ec128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764497.438022:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e248b0 (ffff8b23812ec128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764497.438031:0:51153:0:(vvp_io.c:517:vvp_io_advance()) advancing 32768 bytes 00000080:00200000:1.0:1775764497.438061:0:51153:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x1d:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764497.438066:0:51153:0:(file.c:2206:ll_file_io_generic()) f18.sanity-pcc: 1 io complete with rc: 0, result: 32768, restart: 0 00000080:00200000:1.0:1775764497.438071:0:51153:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 32768 00000080:00200008:1.0:1775764497.438233:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 65536, count: 32768 00000080:00200000:1.0:1775764497.438241:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 16 00000080:00200000:1.0:1775764497.438246:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 17 00000080:00200000:1.0:1775764497.438248:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 18 00000080:00200000:1.0:1775764497.438250:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 19 00000080:00200000:1.0:1775764497.438252:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 20 00000080:00200000:1.0:1775764497.438254:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 21 00000080:00200000:1.0:1775764497.438256:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 22 00000080:00200000:1.0:1775764497.438257:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 23 00000080:00200008:1.0:1775764497.438321:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 98304, count: 32768 00000080:00200000:1.0:1775764497.438323:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 24 00000080:00200000:1.0:1775764497.438325:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 25 00000080:00200000:1.0:1775764497.438327:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 26 00000080:00200000:1.0:1775764497.438329:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 27 00000080:00200000:1.0:1775764497.438330:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 28 00000080:00200000:1.0:1775764497.438332:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 29 00000080:00200000:1.0:1775764497.438334:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 30 00000080:00200000:1.0:1775764497.438335:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 31 00000080:00200008:1.0:1775764497.438397:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 131072, count: 32768 00000080:00200000:1.0:1775764497.438399:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 32 00000080:00200000:1.0:1775764497.438401:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 33 00000080:00200000:1.0:1775764497.438402:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 34 00000080:00200000:1.0:1775764497.438404:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 35 00000080:00200000:1.0:1775764497.438405:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 36 00000080:00200000:1.0:1775764497.438407:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 37 00000080:00200000:1.0:1775764497.438409:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 38 00000080:00200000:1.0:1775764497.438411:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 39 00000080:00200008:1.0:1775764497.438501:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 163840, count: 32768 00000080:00200000:1.0:1775764497.438504:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 40 00000080:00200000:1.0:1775764497.438507:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 41 00000080:00200000:1.0:1775764497.438509:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 42 00000080:00200000:1.0:1775764497.438511:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 43 00000080:00200000:1.0:1775764497.438512:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 44 00000080:00200000:1.0:1775764497.438514:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 45 00000080:00200000:1.0:1775764497.438516:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 46 00000080:00200000:1.0:1775764497.438518:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 47 00000080:00200008:1.0:1775764497.438587:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 196608, count: 32768 00000080:00200000:1.0:1775764497.438590:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 48 00000080:00200000:1.0:1775764497.438593:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 49 00000080:00200000:1.0:1775764497.438597:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 50 00000080:00200000:1.0:1775764497.438601:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 51 00000080:00200000:1.0:1775764497.438604:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 52 00000080:00200000:1.0:1775764497.438606:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 53 00000080:00200000:1.0:1775764497.438609:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 54 00000080:00200000:1.0:1775764497.438612:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 55 00000080:00200008:1.0:1775764497.438680:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 229376, count: 32768 00000080:00200000:1.0:1775764497.438683:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 56 00000080:00200000:1.0:1775764497.438686:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 57 00000080:00200000:1.0:1775764497.438688:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 58 00000080:00200000:1.0:1775764497.438690:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 59 00000080:00200000:1.0:1775764497.438692:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 60 00000080:00200000:1.0:1775764497.438694:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 61 00000080:00200000:1.0:1775764497.438695:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 62 00000080:00200000:1.0:1775764497.438697:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 63 00000080:00200008:1.0:1775764497.438782:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 262144, count: 32768 00000080:00200000:1.0:1775764497.438786:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 64 00000080:00200000:1.0:1775764497.438789:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 65 00000080:00200000:1.0:1775764497.438791:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 66 00000080:00200000:1.0:1775764497.438792:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 67 00000080:00200000:1.0:1775764497.438794:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 68 00000080:00200000:1.0:1775764497.438796:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 69 00000080:00200000:1.0:1775764497.438798:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 70 00000080:00200000:1.0:1775764497.438800:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 71 00000080:00200008:1.0:1775764497.438862:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 294912, count: 32768 00000080:00200000:1.0:1775764497.438864:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 72 00000080:00200000:1.0:1775764497.438867:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 73 00000080:00200000:1.0:1775764497.438869:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 74 00000080:00200000:1.0:1775764497.438870:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 75 00000080:00200000:1.0:1775764497.438872:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 76 00000080:00200000:1.0:1775764497.438874:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 77 00000080:00200000:1.0:1775764497.438875:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 78 00000080:00200000:1.0:1775764497.438877:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 79 00000080:00200008:1.0:1775764497.438938:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 327680, count: 32768 00000080:00200000:1.0:1775764497.438940:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 80 00000080:00200000:1.0:1775764497.438942:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 81 00000080:00200000:1.0:1775764497.438944:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 82 00000080:00200000:1.0:1775764497.438946:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 83 00000080:00200000:1.0:1775764497.438947:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 84 00000080:00200000:1.0:1775764497.438949:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 85 00000080:00200000:1.0:1775764497.438951:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 86 00000080:00200000:1.0:1775764497.438952:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 87 00000080:00200008:1.0:1775764497.439014:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 360448, count: 32768 00000080:00200000:1.0:1775764497.439015:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 88 00000080:00200000:1.0:1775764497.439017:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 89 00000080:00200000:1.0:1775764497.439018:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 90 00000080:00200000:1.0:1775764497.439020:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 91 00000080:00200000:1.0:1775764497.439021:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 92 00000080:00200000:1.0:1775764497.439023:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 93 00000080:00200000:1.0:1775764497.439024:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 94 00000080:00200000:1.0:1775764497.439026:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 95 00000080:00200008:1.0:1775764497.439087:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 393216, count: 32768 00000080:00200000:1.0:1775764497.439089:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 96 00000080:00200000:1.0:1775764497.439091:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 97 00000080:00200000:1.0:1775764497.439093:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 98 00000080:00200000:1.0:1775764497.439095:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 99 00000080:00200000:1.0:1775764497.439096:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 100 00000080:00200000:1.0:1775764497.439211:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 101 00000080:00200000:1.0:1775764497.439215:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 102 00000080:00200000:1.0:1775764497.439217:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 103 00000080:00200008:1.0:1775764497.439282:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 425984, count: 32768 00000080:00200000:1.0:1775764497.439284:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 104 00000080:00200000:1.0:1775764497.439286:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 105 00000080:00200000:1.0:1775764497.439288:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 106 00000080:00200000:1.0:1775764497.439290:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 107 00000080:00200000:1.0:1775764497.439291:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 108 00000080:00200000:1.0:1775764497.439293:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 109 00000080:00200000:1.0:1775764497.439295:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 110 00000080:00200000:1.0:1775764497.439296:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 111 00000080:00200008:1.0:1775764497.439358:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 458752, count: 32768 00000080:00200000:1.0:1775764497.439360:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 112 00000080:00200000:1.0:1775764497.439361:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 113 00000080:00200000:1.0:1775764497.439363:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 114 00000080:00200000:1.0:1775764497.439364:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 115 00000080:00200000:1.0:1775764497.439366:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 116 00000080:00200000:1.0:1775764497.439368:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 117 00000080:00200000:1.0:1775764497.439370:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 118 00000080:00200000:1.0:1775764497.439371:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 119 00000080:00200008:1.0:1775764497.439433:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 491520, count: 32768 00000080:00200000:1.0:1775764497.439434:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 120 00000080:00200000:1.0:1775764497.439435:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 121 00000080:00200000:1.0:1775764497.439437:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 122 00000080:00200000:1.0:1775764497.439439:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 123 00000080:00200000:1.0:1775764497.439440:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 124 00000080:00200000:1.0:1775764497.439442:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 125 00000080:00200000:1.0:1775764497.439444:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 126 00000080:00200000:1.0:1775764497.439445:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 127 00000080:00200008:1.0:1775764497.439578:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 524288, count: 32768 00000080:00200000:1.0:1775764497.439581:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 128 00000080:00200000:1.0:1775764497.439584:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 129 00000080:00200000:1.0:1775764497.439586:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 130 00000080:00200000:1.0:1775764497.439588:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 131 00000080:00200000:1.0:1775764497.439590:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 132 00000080:00200000:1.0:1775764497.439593:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 133 00000080:00200000:1.0:1775764497.439596:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 134 00000080:00200000:1.0:1775764497.439599:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 135 00000080:00200008:1.0:1775764497.439668:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 557056, count: 32768 00000080:00200000:1.0:1775764497.439671:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 136 00000080:00200000:1.0:1775764497.439674:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 137 00000080:00200000:1.0:1775764497.439676:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 138 00000080:00200000:1.0:1775764497.439679:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 139 00000080:00200000:1.0:1775764497.439681:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 140 00000080:00200000:1.0:1775764497.439683:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 141 00000080:00200000:1.0:1775764497.439685:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 142 00000080:00200000:1.0:1775764497.439687:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 143 00000080:00200008:1.0:1775764497.439759:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 589824, count: 32768 00000080:00200000:1.0:1775764497.439762:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 144 00000080:00200000:1.0:1775764497.439765:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 145 00000080:00200000:1.0:1775764497.439767:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 146 00000080:00200000:1.0:1775764497.439770:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 147 00000080:00200000:1.0:1775764497.439772:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 148 00000080:00200000:1.0:1775764497.439775:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 149 00000080:00200000:1.0:1775764497.439778:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 150 00000080:00200000:1.0:1775764497.439780:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 151 00000080:00200008:1.0:1775764497.439842:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 622592, count: 32768 00000080:00200000:1.0:1775764497.439844:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 152 00000080:00200000:1.0:1775764497.439846:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 153 00000080:00200000:1.0:1775764497.439848:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 154 00000080:00200000:1.0:1775764497.439850:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 155 00000080:00200000:1.0:1775764497.439852:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 156 00000080:00200000:1.0:1775764497.439854:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 157 00000080:00200000:1.0:1775764497.439855:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 158 00000080:00200000:1.0:1775764497.439857:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 159 00000080:00200008:1.0:1775764497.439918:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 655360, count: 32768 00000080:00200000:1.0:1775764497.439919:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 160 00000080:00200000:1.0:1775764497.439921:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 161 00000080:00200000:1.0:1775764497.439924:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 162 00000080:00200000:1.0:1775764497.439926:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 163 00000080:00200000:1.0:1775764497.439928:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 164 00000080:00200000:1.0:1775764497.439930:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 165 00000080:00200000:1.0:1775764497.439932:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 166 00000080:00200000:1.0:1775764497.439934:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 167 00000080:00200008:1.0:1775764497.439996:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 688128, count: 32768 00000080:00200000:1.0:1775764497.439998:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 168 00000080:00200000:1.0:1775764497.440001:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 169 00000080:00200000:1.0:1775764497.440003:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 170 00000080:00200000:1.0:1775764497.440005:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 171 00000080:00200000:1.0:1775764497.440007:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 172 00000080:00200000:1.0:1775764497.440009:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 173 00000080:00200000:1.0:1775764497.440010:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 174 00000080:00200000:1.0:1775764497.440012:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 175 00000080:00200008:1.0:1775764497.440074:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 720896, count: 32768 00000080:00200000:1.0:1775764497.440075:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 176 00000080:00200000:1.0:1775764497.440077:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 177 00000080:00200000:1.0:1775764497.440080:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 178 00000080:00200000:1.0:1775764497.440082:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 179 00000080:00200000:1.0:1775764497.440084:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 180 00000080:00200000:1.0:1775764497.440086:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 181 00000080:00200000:1.0:1775764497.440088:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 182 00000080:00200000:1.0:1775764497.440090:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 183 00000080:00200008:1.0:1775764497.440214:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 753664, count: 32768 00000080:00200000:1.0:1775764497.440219:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 184 00000080:00200000:1.0:1775764497.440222:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 185 00000080:00200000:1.0:1775764497.440224:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 186 00000080:00200000:1.0:1775764497.440225:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 187 00000080:00200000:1.0:1775764497.440227:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 188 00000080:00200000:1.0:1775764497.440229:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 189 00000080:00200000:1.0:1775764497.440231:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 190 00000080:00200000:1.0:1775764497.440232:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 191 00000080:00200008:1.0:1775764497.440294:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 786432, count: 32768 00000080:00200000:1.0:1775764497.440296:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 192 00000080:00200000:1.0:1775764497.440298:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 193 00000080:00200000:1.0:1775764497.440300:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 194 00000080:00200000:1.0:1775764497.440302:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 195 00000080:00200000:1.0:1775764497.440303:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 196 00000080:00200000:1.0:1775764497.440305:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 197 00000080:00200000:1.0:1775764497.440307:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 198 00000080:00200000:1.0:1775764497.440309:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 199 00000080:00200008:1.0:1775764497.440379:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 819200, count: 32768 00000080:00200000:1.0:1775764497.440381:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 200 00000080:00200000:1.0:1775764497.440383:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 201 00000080:00200000:1.0:1775764497.440385:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 202 00000080:00200000:1.0:1775764497.440387:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 203 00000080:00200000:1.0:1775764497.440388:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 204 00000080:00200000:1.0:1775764497.440390:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 205 00000080:00200000:1.0:1775764497.440392:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 206 00000080:00200000:1.0:1775764497.440393:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 207 00000080:00200008:1.0:1775764497.440477:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 851968, count: 32768 00000080:00200000:1.0:1775764497.440480:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 208 00000080:00200000:1.0:1775764497.440482:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 209 00000080:00200000:1.0:1775764497.440484:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 210 00000080:00200000:1.0:1775764497.440486:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 211 00000080:00200000:1.0:1775764497.440488:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 212 00000080:00200000:1.0:1775764497.440490:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 213 00000080:00200000:1.0:1775764497.440491:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 214 00000080:00200000:1.0:1775764497.440493:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 215 00000080:00200008:1.0:1775764497.440728:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 884736, count: 32768 00000080:00200000:1.0:1775764497.440735:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 216 00000080:00200000:1.0:1775764497.440741:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 217 00000080:00200000:1.0:1775764497.440745:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 218 00000080:00200000:1.0:1775764497.440748:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 219 00000080:00200000:1.0:1775764497.440751:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 220 00000080:00200000:1.0:1775764497.440754:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 221 00000080:00200000:1.0:1775764497.440756:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 222 00000080:00200000:1.0:1775764497.440759:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 223 00000080:00200008:1.0:1775764497.440828:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 917504, count: 32768 00000080:00200000:1.0:1775764497.440832:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 224 00000080:00200000:1.0:1775764497.440837:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 225 00000080:00200000:1.0:1775764497.440840:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 226 00000080:00200000:1.0:1775764497.440843:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 227 00000080:00200000:1.0:1775764497.440846:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 228 00000080:00200000:1.0:1775764497.440848:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 229 00000080:00200000:1.0:1775764497.440851:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 230 00000080:00200000:1.0:1775764497.440854:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 231 00000080:00200008:1.0:1775764497.440922:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 950272, count: 32768 00000080:00200000:1.0:1775764497.440926:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 232 00000080:00200000:1.0:1775764497.440931:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 233 00000080:00200000:1.0:1775764497.440934:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 234 00000080:00200000:1.0:1775764497.440938:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 235 00000080:00200000:1.0:1775764497.440941:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 236 00000080:00200000:1.0:1775764497.440944:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 237 00000080:00200000:1.0:1775764497.440947:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 238 00000080:00200000:1.0:1775764497.440950:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 239 00000080:00200008:1.0:1775764497.441019:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 983040, count: 32768 00000080:00200000:1.0:1775764497.441024:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 240 00000080:00200000:1.0:1775764497.441028:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 241 00000080:00200000:1.0:1775764497.441031:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 242 00000080:00200000:1.0:1775764497.441034:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 243 00000080:00200000:1.0:1775764497.441037:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 244 00000080:00200000:1.0:1775764497.441040:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 245 00000080:00200000:1.0:1775764497.441043:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 246 00000080:00200000:1.0:1775764497.441046:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 247 00000080:00200008:1.0:1775764497.441114:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 1015808, count: 32768 00000080:00200000:1.0:1775764497.441119:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 248 00000080:00200000:1.0:1775764497.441123:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 249 00000080:00200000:1.0:1775764497.441126:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 250 00000080:00200000:1.0:1775764497.441129:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 251 00000080:00200000:1.0:1775764497.441133:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 252 00000080:00200000:1.0:1775764497.441136:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 253 00000080:00200000:1.0:1775764497.441140:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 254 00000080:00200000:1.0:1775764497.441143:0:51153:0:(rw.c:1928:ll_readpage()) fast read pgno: 255 00000080:00200008:1.0:1775764497.441211:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 1048576, count: 32768 00000080:00200000:1.0:1775764497.441261:0:51153:0:(file.c:2058:ll_file_io_generic()) f18.sanity-pcc: read ppos: 1049600, bytes: 31744 00000080:00200000:1.0:1775764497.441267:0:51153:0:(file.c:1815:ll_io_set_mirror()) f18.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764497.441276:0:51153:0:(cl_io.c:214:cl_io_rw_init()) header@00000000c393be83[0x110, 258, [0x200003ab1:0x1d:0x0] hash] 00000020:00200000:1.0:1775764497.441278:0:51153:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [1049600, 1081344) 0 0 00000080:00200000:1.0:1775764497.441283:0:51153:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x1d:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764497.441303:0:51153:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 1049600 chunk: [1049600, 1081344) 1081344, 31744 00020000:00200000:1.0:1775764497.441307:0:51153:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764497.441340:0:51153:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [256, 263] 00000008:00010000:1.0:1775764497.441356:0:51153:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764497.441361:0:51153:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318c30, osc lock ffff8b2382cf6720, flags 0x0 00010000:00010000:1.0:1775764497.441386:0:51153:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1775764497.441399:0:51153:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 1081343) ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1775764497.441410:0:51153:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1775764497.441416:0:51153:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1775764497.441421:0:51153:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000080:00200000:1.0:1775764497.441428:0:51153:0:(vvp_io.c:826:vvp_io_read_start()) f18.sanity-pcc: read [1049600, 1081344) 00000001:00010000:1.0:1775764497.441441:0:51153:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab1:0x1d:0x0] 00000008:00010000:1.0:1775764497.441444:0:51153:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764497.441445:0:51153:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318230, osc lock ffff8b2382cf77c0, flags 0x400001002 00010000:00010000:1.0:1775764497.441450:0:51153:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1775764497.441500:0:51153:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000020:00200000:1.0:1775764497.441509:0:51153:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764497.441512:0:51153:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 8/1,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1775764497.441519:0:51153:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 7/1,0 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000020:00200000:1.0:1775764497.441527:0:51153:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764497.441534:0:51153:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x1d:0x0] updating i_size 1049600 i_blocks 2055 00000020:00010000:1.0:1775764497.441538:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1bd98 (ffff8b2387cba128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764497.441541:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1bd98 (ffff8b2387cba128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764497.441544:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318230 (ffff8b2387cba3f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764497.441548:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1bd98 (ffff8b2387cba128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764497.441550:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318230 (ffff8b2387cba128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764497.441561:0:51153:0:(vvp_io.c:885:vvp_io_read_start()) count:31744 iocb pos:1049600 00000020:00010000:1.0:1775764497.441569:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb74e8 (ffff8b2387cba128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764497.441572:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb74e8 (ffff8b2387cba128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764497.441574:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318c30 (ffff8b2387cba3f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764497.441577:0:51153:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1775764497.441674:0:51153:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000020:00010000:1.0:1775764497.441689:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb74e8 (ffff8b2387cba128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764497.441692:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318c30 (ffff8b2387cba128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764497.441700:0:51153:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:1.0:1775764497.441731:0:51153:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x1d:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764497.441734:0:51153:0:(file.c:2206:ll_file_io_generic()) f18.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:1.0:1775764497.441736:0:51153:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:1.0:1775764497.441748:0:51153:0:(file.c:2454:do_file_read_iter()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], ppos: 1049600, count: 24576 00000080:00200000:1.0:1775764497.441759:0:51153:0:(file.c:2058:ll_file_io_generic()) f18.sanity-pcc: read ppos: 1049600, bytes: 24576 00000080:00200000:1.0:1775764497.441761:0:51153:0:(file.c:1815:ll_io_set_mirror()) f18.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764497.441764:0:51153:0:(cl_io.c:214:cl_io_rw_init()) header@00000000c393be83[0x110, 258, [0x200003ab1:0x1d:0x0] hash] 00000020:00200000:1.0:1775764497.441765:0:51153:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [1049600, 1074176) 0 0 00000080:00200000:1.0:1775764497.441767:0:51153:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x1d:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764497.441772:0:51153:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 1049600 chunk: [1049600, 1074176) 1074176, 24576 00020000:00200000:1.0:1775764497.441774:0:51153:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764497.441781:0:51153:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [256, 262] 00000008:00010000:1.0:1775764497.441785:0:51153:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764497.441787:0:51153:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318c30, osc lock ffff8b2382cf6720, flags 0x0 00010000:00010000:1.0:1775764497.441793:0:51153:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764497.441799:0:51153:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 1077247) ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764497.441805:0:51153:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764497.441810:0:51153:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764497.441814:0:51153:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000080:00200000:1.0:1775764497.441818:0:51153:0:(vvp_io.c:826:vvp_io_read_start()) f18.sanity-pcc: read [1049600, 1074176) 00000001:00010000:1.0:1775764497.441936:0:51153:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab1:0x1d:0x0] 00000008:00010000:1.0:1775764497.441940:0:51153:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764497.441941:0:51153:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318230, osc lock ffff8b2382cf77c0, flags 0x400001002 00010000:00010000:1.0:1775764497.441945:0:51153:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764497.441950:0:51153:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000020:00200000:1.0:1775764497.441957:0:51153:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764497.441960:0:51153:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 8/1,1 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764497.441967:0:51153:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 7/1,0 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000020:00200000:1.0:1775764497.441973:0:51153:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764497.441977:0:51153:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x1d:0x0] updating i_size 1049600 i_blocks 2055 00000020:00010000:1.0:1775764497.441980:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1bd98 (ffff8b2387cba128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764497.441982:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1bd98 (ffff8b2387cba128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764497.441984:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318230 (ffff8b2387cba3f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764497.441987:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1bd98 (ffff8b2387cba128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764497.441989:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318230 (ffff8b2387cba128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764497.441994:0:51153:0:(vvp_io.c:885:vvp_io_read_start()) count:24576 iocb pos:1049600 00000020:00010000:1.0:1775764497.441998:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb74e8 (ffff8b2387cba128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764497.442001:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb74e8 (ffff8b2387cba128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764497.442003:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318c30 (ffff8b2387cba3f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764497.442007:0:51153:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764497.442014:0:51153:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00010000:1.0:1775764497.442023:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb74e8 (ffff8b2387cba128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764497.442025:0:51153:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318c30 (ffff8b2387cba128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764497.442029:0:51153:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:1.0:1775764497.442040:0:51153:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x1d:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764497.442043:0:51153:0:(file.c:2206:ll_file_io_generic()) f18.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:1.0:1775764497.442046:0:51153:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:1.0:1775764497.442064:0:51153:0:(file.c:5170:ll_file_seek()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], offset: 0, type: SEEK_CUR 00000080:00200008:1.0:1775764497.442068:0:51153:0:(file.c:5201:ll_file_seek()) COMPLETED file f18.sanity-pcc:[0x200003ab1:0x1d:0x0], offset: 0, type: SEEK_CUR, rc = 1049600 00000080:00200008:1.0:1775764497.442083:0:51153:0:(file.c:447:ll_file_release()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0](ffff8b2390ec4b08), flags 100000 00000002:00100000:1.0:1775764497.442150:0:51153:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fd500 x1862022098373760/t34359738673(34359738673) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764513 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'md5sum.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764497.442163:0:51153:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b238a3bf000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764497.442170:0:51153:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fdc00 x1862022098374272/t0(0) o35->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:23/10 lens 392/752 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764497.442191:0:51153:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job md5sum:82781e18-e0e3-4f9f-9a2b-041bf0344414:51153:1862022098374272:192.168.202.128@tcp:35:md5sum.0 00000100:00100000:1.0:1775764497.442281:0:51153:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6f840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764497.444679:0:51153:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job md5sum:82781e18-e0e3-4f9f-9a2b-041bf0344414:51153:1862022098374272:192.168.202.128@tcp:35:md5sum.0 00000002:00100000:1.0:1775764497.444719:0:51153:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fd500 x1862022098373760/t34359738673(34359738673) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764513 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'md5sum.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764497.469182:0:51158:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764497.469206:0:51158:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764497.469262:0:51158:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 51158 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.469270:0:51158:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 51158 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.469278:0:51158:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 51158 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764497.469299:0:51158:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:51158:1862022098374400:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764497.469371:0:51158:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6f840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764497.472002:0:51158:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:51158:1862022098374400:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764497.472024:0:51158:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.472033:0:51158:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.472046:0:51158:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.472056:0:51158:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.472069:0:51158:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.472077:0:51158:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.472085:0:51158:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764497.472096:0:51158:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864fdc00 x1862022098374400/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764513 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764497.472118:0:51158:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.472130:0:51158:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1775764497.472135:0:51158:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764497.472161:0:51158:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 8704 00000080:00010000:1.0:1775764497.472166:0:51158:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b17c 00000080:00010000:1.0:1775764497.472169:0:51158:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764497.472172:0:51158:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b17c from it ffffa60b01edbb98 00010000:00010000:1.0:1775764497.472174:0:51158:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.472181:0:51158:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764497.472203:0:51158:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764497.472229:0:51158:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=f18.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags=8 00000080:00200000:1.0:1775764497.472233:0:51158:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f18.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=getattr 00000002:00010000:1.0:1775764497.472252:0:51158:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f18.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764497.472297:0:51158:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 51158 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.472305:0:51158:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 51158 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.472315:0:51158:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 51158 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764497.472337:0:51158:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:51158:1862022098374528:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764497.472427:0:51158:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6f840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764497.482138:0:51158:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:51158:1862022098374528:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764497.482175:0:51158:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4e2 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.482204:0:51158:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 4/1,0 mode: --/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4e2 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.482213:0:51158:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 4/1,0 mode: --/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4e2 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.482226:0:51158:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f4e2 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.482238:0:51158:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4e2 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.482253:0:51158:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 5/2,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f4e2 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.482261:0:51158:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 5/2,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f4e2 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.482291:0:51158:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f4e2 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764497.482307:0:51158:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864fdc00 x1862022098374528/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 584/640 e 0 to 0 dl 1775764513 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764497.482321:0:51158:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4e2 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764497.482335:0:51158:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4e2 expref: -99 pid: 51158 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.482348:0:51158:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 4 res 8589949617/29 (0 0) 00000080:00200000:1.0:1775764497.482355:0:51158:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764497.482479:0:51158:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x1d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764497.482490:0:51158:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x1d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764497.482583:0:51158:0:(namei.c:147:ll_iget()) got inode: [0x200003ab1:0x1d:0x0](ffff8b2389a65b88): rc = 0 00000080:00200000:1.0:1775764497.488094:0:51158:0:(file.c:6766:ll_layout_conf()) [0x200003ab1:0x1d:0x0]: layout version change: 4294967294 -> 2 00000080:00010000:1.0:1775764497.488157:0:51158:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x1d:0x0](ffff8b2389a65b88) for lock 0x73cfd86cfc17b183 00000080:00010000:1.0:1775764497.488177:0:51158:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x1d:0x0](ffff8b2389a65b88) 00000080:00010000:1.0:1775764497.488181:0:51158:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b183 from it ffffa60b01edbb60 00010000:00010000:1.0:1775764497.488187:0:51158:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0304f4e2 expref: -99 pid: 51158 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.488201:0:51158:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304f4e2 expref: -99 pid: 51158 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:1.0:1775764497.917301:0:51188:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764497.917319:0:51188:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764497.917342:0:51188:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764497.917362:0:51188:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764497.917380:0:51188:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764497.917389:0:51188:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764497.917399:0:51188:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764497.917403:0:51188:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b17c 00000080:00010000:1.0:1775764497.917413:0:51188:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b17c from it ffffa60b01f0bcb8 00010000:00010000:1.0:1775764497.917416:0:51188:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764497.917430:0:51188:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764497.917500:0:51188:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764497.917508:0:51188:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764497.917514:0:51188:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764497.917522:0:51188:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764497.917528:0:51188:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764497.917531:0:51188:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764497.917534:0:51188:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b17c 00000080:00010000:1.0:1775764497.917538:0:51188:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b17c from it ffffa60b01f0bb58 00010000:00010000:1.0:1775764497.917540:0:51188:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764497.917548:0:51188:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764497.917556:0:51188:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764497.917563:0:51188:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f18.sanity-pcc, flags=8 00000080:00200008:1.0:1775764497.917569:0:51188:0:(file.c:6226:ll_getattr_dentry()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0](ffff8b2389a65b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764497.917573:0:51188:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab1:0x1d:0x0](ffff8b2389a65b88),name=f18.sanity-pcc 00000002:00010000:1.0:1775764497.917583:0:51188:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x1d:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764497.917591:0:51188:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304f4e2 expref: -99 pid: 51158 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764497.917599:0:51188:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304f4e2 expref: -99 pid: 51158 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764497.917605:0:51188:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764497.917609:0:51188:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x1d:0x0](ffff8b2389a65b88) 00000080:00010000:1.0:1775764497.917612:0:51188:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x1d:0x0](ffff8b2389a65b88) for lock 0x73cfd86cfc17b183 00000080:00010000:1.0:1775764497.917616:0:51188:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b183 from it ffffa60b01f0bcb8 00010000:00010000:1.0:1775764497.917618:0:51188:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0304f4e2 expref: -99 pid: 51158 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764497.917626:0:51188:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304f4e2 expref: -99 pid: 51158 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764497.917711:0:51188:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x1d:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764497.917722:0:51188:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab1:0x1d:0x0] 00000008:00010000:1.0:1775764497.917749:0:51188:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764497.917753:0:51188:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318930, osc lock ffff8b2382cf6720, flags 0x400001002 00010000:00010000:1.0:1775764497.917812:0:51188:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 0000000073fdd2a9/0x73cfd86cfc17b18a lrc: 3/1,0 mode: --/PR res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 51188 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.917822:0:51188:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 0000000073fdd2a9/0x73cfd86cfc17b18a lrc: 3/1,0 mode: --/PR res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 51188 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1775764497.917859:0:51188:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:1.0:1775764497.924268:0:46140:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764497.924305:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118976512 00000100:00100000:1.0:1775764497.924318:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118976512 00000100:00100000:1.0:1775764497.924388:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 86 00000100:00100000:1.0:1775764497.924399:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b238a0f0c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43212:x1862022118976512:12345-192.168.202.128@tcp:106: 00010000:00010000:1.0:1775764497.924417:0:46140:0:(ldlm_lockd.c:2525:ldlm_callback_handler()) ### glimpse ast ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764497.924432:0:46140:0:(ldlm_lockd.c:2116:ldlm_handle_gl_callback()) ### client glimpse AST callback handler ns: lustre-OST0000-osc-ffff8b238a3bf000 lock: 000000009e189fb3/0x73cfd86cfc17b167 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304f479 expref: -99 pid: 51150 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000020:00010000:1.0:1775764497.924527:0:46140:0:(cl_object.c:330:cl_object_glimpse()) header@00000000c393be83[0x110, 258, [0x200003ab1:0x1d:0x0] hash] 00000020:00010000:1.0:1775764497.924529:0:46140:0:(cl_object.c:330:cl_object_glimpse()) size: 1049600 mtime: 1775764497 atime: 1775764497 ctime: 1775764497 blocks: 2055 00000020:00010000:1.0:1775764497.924532:0:46140:0:(cl_object.c:330:cl_object_glimpse()) header@000000007d0ddfdc[0x110, 3, [0x240000400:0x108:0x0] hash] 00000020:00010000:1.0:1775764497.924533:0:46140:0:(cl_object.c:330:cl_object_glimpse()) size: 1049600 mtime: 1775764497 atime: 1775764497 ctime: 1775764497 blocks: 2055 00000100:00100000:1.0:1775764497.924604:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b238a0f0c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43212:x1862022118976512:12345-192.168.202.128@tcp:106: Request processed in 207us (1297us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764497.924608:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 86 00000100:00100000:1.0:1775764497.927849:0:2376:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b2384b87800: committing for last_committed 34359738395 gen 2 00010000:00010000:1.0:1775764497.927886:0:2376:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 0000000073fdd2a9/0x73cfd86cfc17b18a lrc: 6/2,0 mode: --/PR res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 51188 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.927899:0:2376:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 0000000073fdd2a9/0x73cfd86cfc17b18a lrc: 6/2,0 mode: --/PR res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 51188 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.927911:0:2376:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 0000000073fdd2a9/0x73cfd86cfc17b18a lrc: 6/2,0 mode: --/PR res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 51188 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.927925:0:2376:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 0000000073fdd2a9/0x73cfd86cfc17b18a lrc: 6/2,0 mode: --/PR res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 51188 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.927933:0:2376:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 0000000073fdd2a9/0x73cfd86cfc17b18a lrc: 5/1,0 mode: --/PR res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 51188 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1775764497.927952:0:2376:0:(osc_lock.c:187:osc_lock_lvb_update()) ### no dlm lock: acquired size 1049600, setting rss=1049600; leaving kms=0, end=18446744073709551615 00000020:00200000:1.0:1775764497.927961:0:2376:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1775764497.927970:0:2376:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 0000000073fdd2a9/0x73cfd86cfc17b18a lrc: 4/1,0 mode: --/PR res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 51188 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.927978:0:2376:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 0000000073fdd2a9/0x73cfd86cfc17b18a lrc: 3/0,0 mode: --/PR res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 51188 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.927987:0:2376:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 0000000073fdd2a9/0x73cfd86cfc17b18a lrc: 4/0,0 mode: --/PR res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 51188 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.927994:0:2376:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23bacb4c00 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:1.0:1775764497.928000:0:2376:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 0000000073fdd2a9/0x73cfd86cfc17b18a lrc: 5/0,0 mode: --/PR res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 51188 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.928055:0:2376:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764497.928074:0:2376:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 0000000073fdd2a9/0x73cfd86cfc17b18a lrc: 3/0,0 mode: --/PR res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 51188 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764497.928089:0:2376:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 0000000073fdd2a9/0x73cfd86cfc17b18a lrc: 0/0,0 mode: --/PR res: [0x240000400:0x108:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 51188 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764497.928117:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098374656:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764497.936282:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118976640 00000100:00100000:1.0:1775764497.936301:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118976640 00000100:00100000:1.0:1775764497.936352:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118976768 00000100:00100000:1.0:1775764497.936356:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118976768 00000100:00100000:1.0:1775764497.936379:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 87 00000100:00100000:1.0:1775764497.936389:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dac380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42326:x1862022118976640:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764497.936412:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b183 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0304f4e2 expref: -99 pid: 51158 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764497.936534:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764497.936542:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dac380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42326:x1862022118976640:12345-192.168.202.128@tcp:104: Request processed in 154us (2709us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764497.936549:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 87 00000100:00100000:1.0:1775764497.936566:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 88 00000100:00100000:1.0:1775764497.936572:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dac700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42326:x1862022118976768:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764497.936580:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23bacb4400/0x73cfd86cfc17b175 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0304f4b1 expref: -99 pid: 51153 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764497.936662:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764497.936668:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dac700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42326:x1862022118976768:12345-192.168.202.128@tcp:104: Request processed in 96us (2767us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764497.936675:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 88 00000100:00100000:1.0:1775764497.977695:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764498.003568:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764498.055254:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:1.0:1775764498.064831:0:51188:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9ceb800/0x73cfd86cfc17b198 lrc: 2/0,0 mode: PW/PW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x57b8ff0e0304f50c expref: -99 pid: 51188 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764498.064853:0:51188:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9ceb800/0x73cfd86cfc17b198 lrc: 0/0,0 mode: --/PW res: [0x200003ab1:0x1d:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x57b8ff0e0304f50c expref: -99 pid: 51188 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764498.064873:0:51188:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea5500 x1862022098375424/t34359738676(34359738676) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764513 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764498.064885:0:51188:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764498.064890:0:51188:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fdf80 x1862022098376704/t0(0) o35->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:23/10 lens 504/752 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764498.064912:0:51188:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:51188:1862022098376704:192.168.202.128@tcp:35:lfs.0 00000100:00100000:1.0:1775764498.064991:0:51188:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6f3c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764498.070927:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764498.070968:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764498.071000:0:46139:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764498.071016:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118977408 00000100:00100000:1.0:1775764498.071025:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118977408 00000100:00100000:1.0:1775764498.071075:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118977536 00000100:00100000:1.0:1775764498.071078:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118977536 00000100:00100000:1.0:1775764498.071093:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 92 00000100:00100000:1.0:1775764498.071102:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23864fd180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42328:x1862022118977408:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764498.071123:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9cea200/0x73cfd86cfc17b19f lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0304f51a expref: -99 pid: 51188 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764498.071212:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764498.071219:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23864fd180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42328:x1862022118977408:12345-192.168.202.128@tcp:104: Request processed in 118us (302us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764498.071227:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 92 00000100:00100000:1.0:1775764498.071244:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 93 00000100:00100000:1.0:1775764498.071250:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b238a397850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42328:x1862022118977536:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764498.071258:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b9c5f000/0x73cfd86cfc17b128 lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0304f3a7 expref: -99 pid: 51150 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 1 00010000:00010000:1.0:1775764498.071292:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1775764498.071298:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b238a397850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42328:x1862022118977536:12345-192.168.202.128@tcp:104: Request processed in 48us (330us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764498.071303:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 93 00000080:00200008:1.0:1775764498.551874:0:51217:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764498.551891:0:51217:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764498.551914:0:51217:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764498.551932:0:51217:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764498.551949:0:51217:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764498.551957:0:51217:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764498.551965:0:51217:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764498.551969:0:51217:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b17c 00000080:00010000:1.0:1775764498.551977:0:51217:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b17c from it ffffa60b01fabcb8 00010000:00010000:1.0:1775764498.551981:0:51217:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764498.551989:0:51217:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764498.552016:0:51217:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764498.552021:0:51217:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764498.552027:0:51217:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764498.552035:0:51217:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764498.552335:0:51217:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764498.552340:0:51217:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764498.552343:0:51217:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b17c 00000080:00010000:1.0:1775764498.552347:0:51217:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b17c from it ffffa60b01fabb58 00010000:00010000:1.0:1775764498.552350:0:51217:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764498.552358:0:51217:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764498.552367:0:51217:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764498.552374:0:51217:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f18.sanity-pcc, flags=8 00000080:00200008:1.0:1775764498.552381:0:51217:0:(file.c:6226:ll_getattr_dentry()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0](ffff8b2389a65b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764498.552385:0:51217:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab1:0x1d:0x0](ffff8b2389a65b88),name=f18.sanity-pcc 00000002:00010000:1.0:1775764498.552396:0:51217:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x1d:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764498.552405:0:51217:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51600/0x73cfd86cfc17b1b4 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304f552 expref: -99 pid: 51202 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764498.552413:0:51217:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51600/0x73cfd86cfc17b1b4 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304f552 expref: -99 pid: 51202 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764498.552419:0:51217:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764498.552423:0:51217:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x1d:0x0](ffff8b2389a65b88) 00000080:00010000:1.0:1775764498.552425:0:51217:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x1d:0x0](ffff8b2389a65b88) for lock 0x73cfd86cfc17b1b4 00000080:00010000:1.0:1775764498.552428:0:51217:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b1b4 from it ffffa60b01fabcb8 00010000:00010000:1.0:1775764498.552430:0:51217:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51600/0x73cfd86cfc17b1b4 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0304f552 expref: -99 pid: 51202 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764498.552437:0:51217:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51600/0x73cfd86cfc17b1b4 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x1d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304f552 expref: -99 pid: 51202 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764498.552844:0:51217:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764498.552853:0:51217:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764498.552861:0:51217:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764498.552869:0:51217:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764498.552875:0:51217:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764498.552880:0:51217:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764498.552882:0:51217:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b17c 00000080:00010000:1.0:1775764498.552886:0:51217:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b17c from it ffffa60b01fabbf8 00010000:00010000:1.0:1775764498.552889:0:51217:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764498.552895:0:51217:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764498.552904:0:51217:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764498.552907:0:51217:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f18.sanity-pcc, flags=265 00000080:00200000:1.0:1775764498.552911:0:51217:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab1:0x1d:0x0](ffff8b2389a65b88), inode mode 81a4 mask 46 00000080:00200008:1.0:1775764498.552919:0:51217:0:(file.c:1057:ll_file_open()) START file f18.sanity-pcc:[0x200003ab1:0x1d:0x0](ffff8b2389a65b88), flags 154402 00000002:00010000:1.0:1775764498.552931:0:51217:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x1d:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240114403 00010000:00010000:1.0:1775764498.553169:0:51217:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b1c2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 51217 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764498.553177:0:51217:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b1c2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 51217 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764498.553190:0:51217:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764498.553298:0:51217:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fdc00 x1862022098377984/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/131984 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764498.553308:0:51217:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4000/0x73cfd86cfc17b1c2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 51217 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764498.553333:0:51217:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:51217:1862022098377984:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764498.553801:0:51217:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6fcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764498.559937:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764498.573247:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098378112:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764499.343838:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200080:1.0:1775764499.351184:0:51218:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=c0086696 arg=7f2628000d20 00800000:00000080:1.0:1775764499.351218:0:51218:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=c0086696 len=4128 karg=ffff8b2384b0a000 uarg=0000000000000000 00000002:00000080:1.0:1775764499.351228:0:51218:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2384b87800: cmd=c0086696 len=4128 karg=ffff8b2384b0a000 uarg=0000000000000000 00000002:00000080:1.0:1775764499.351235:0:51218:0:(mdc_request.c:1738:mdc_ioc_fid2path()) path get [0x200003ab1:0x1d:0x0] from 18446744073709551615 #0 00000100:00100000:1.0:1775764499.351289:0:51218:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51218:1862022098379008:192.168.202.128@tcp:53:lhsmtool_posix.0 00000100:00100000:1.0:1775764499.351532:0:51218:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6f000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764499.357823:0:51218:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51218:1862022098379008:192.168.202.128@tcp:53:lhsmtool_posix.0 00000002:00000080:1.0:1775764499.357879:0:51218:0:(mdc_request.c:1758:mdc_ioc_fid2path()) path got [0x200000007:0x1:0x0] from 18446744073709551615 #0: f18.sanity-pcc 00000080:00200080:1.0:1775764499.358015:0:51218:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=c01066f8 arg=16039e8 00800000:00000080:1.0:1775764499.358027:0:51218:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=c01066f8 len=16 karg=ffffa60b01fb3cf0 uarg=ffffa60b01fb3cc0 00000080:00200080:1.0:1775764499.358156:0:51218:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000002:0x2:0x0](ffff8b238990e3c8) cmd=c1406916 arg=7f2628000d20 00000100:00100000:1.0:1775764499.358221:0:51218:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51218:1862022098379136:192.168.202.128@tcp:34:lhsmtool_posix.0 00000100:00100000:1.0:1775764499.358758:0:51218:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6f000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764499.365545:0:51218:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51218:1862022098379136:192.168.202.128@tcp:34:lhsmtool_posix.0 00000080:00200000:1.0:1775764499.366952:0:51218:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000002:0x2:0x0](ffff8b238990e3c8), inode mode 4040 mask 1 00000080:00200000:1.0:1775764499.366978:0:51218:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=0x200003ab1:0x1d:0x0, dir=[0x200000002:0x2:0x0](ffff8b238990e3c8), file ffff8b23a01a6180, open_flags 68800, mode 0 opened 0 00000080:00200000:1.0:1775764499.366987:0:51218:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=0x200003ab1:0x1d:0x0, dir=[0x200000002:0x2:0x0](ffff8b238990e3c8), intent=open 00000002:00010000:1.0:1775764499.367016:0:51218:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x1d:0x0]) in obj [0x200000002:0x2:0x0], intent: open flags 041504001 00010000:00010000:1.0:1775764499.367088:0:51218:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4200/0x73cfd86cfc17b1d0 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 51218 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764499.367105:0:51218:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4200/0x73cfd86cfc17b1d0 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 51218 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764499.367119:0:51218:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764499.367130:0:51218:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd180 x1862022098379392/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/131984 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764499.367140:0:51218:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4200/0x73cfd86cfc17b1d0 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 51218 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764499.367165:0:51218:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51218:1862022098379392:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764499.367358:0:51218:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6f000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764499.372927:0:51218:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51218:1862022098379392:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:1.0:1775764499.372955:0:51218:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4200/0x73cfd86cfc17b1d0 lrc: 4/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 51218 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764499.372968:0:51218:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4200/0x73cfd86cfc17b1d0 lrc: 4/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 51218 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764499.372975:0:51218:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4200/0x73cfd86cfc17b1d0 lrc: 4/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 51218 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764499.372982:0:51218:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4200/0x73cfd86cfc17b1d0 lrc: 3/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 51218 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764499.372992:0:51218:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4200/0x73cfd86cfc17b1d0 lrc: 4/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 51218 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764499.372998:0:51218:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23bacb4200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764499.373004:0:51218:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4200/0x73cfd86cfc17b1d0 lrc: 4/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 51218 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764499.373011:0:51218:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4200/0x73cfd86cfc17b1d0 lrc: 4/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 51218 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764499.373021:0:51218:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4200/0x73cfd86cfc17b1d0 lrc: 5/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 51218 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764499.373781:0:51218:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764499.373794:0:51218:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4200/0x73cfd86cfc17b1d0 lrc: 3/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 51218 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764499.373803:0:51218:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4200/0x73cfd86cfc17b1d0 lrc: 0/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 51218 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764499.373827:0:51218:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23864fd180 x1862022098379392/t34359738689(34359738689) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/664 e 0 to 0 dl 1775764515 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764499.373841:0:51218:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fd180 x1862022098379392/t34359738689(34359738689) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/664 e 0 to 0 dl 1775764515 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764499.373860:0:51218:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000002:0x2:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764499.374000:0:51218:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab1:0x1d:0x0], updating i_size 1049600 00000080:00200000:1.0:1775764499.374004:0:51218:0:(namei.c:147:ll_iget()) got inode: [0x200003ab1:0x1d:0x0](ffff8b2389a65b88): rc = 0 00000080:00010000:1.0:1775764499.374010:0:51218:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x1d:0x0](ffff8b2389a65b88) for lock 0x0 00000080:00200008:1.0:1775764499.375059:0:51218:0:(file.c:1057:ll_file_open()) START file 0x200003ab1:0x1d:0x0:[0x200003ab1:0x1d:0x0](ffff8b2389a65b88), flags 1504000 00000002:00100000:1.0:1775764499.375096:0:51218:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fd180 x1862022098379392/t34359738689(34359738689) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764515 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764499.375188:0:51218:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab1:0x1d:0x0](ffff8b2389a65b88), inode mode 81a4 mask 44 00000080:00200080:1.0:1775764499.375233:0:51218:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=c0086696 arg=7f2628000d20 00800000:00000080:1.0:1775764499.375255:0:51218:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=c0086696 len=4128 karg=ffff8b2384b0a000 uarg=0000000000000000 00000002:00000080:1.0:1775764499.375263:0:51218:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2384b87800: cmd=c0086696 len=4128 karg=ffff8b2384b0a000 uarg=0000000000000000 00000002:00000080:1.0:1775764499.375268:0:51218:0:(mdc_request.c:1738:mdc_ioc_fid2path()) path get [0x200003ab1:0x1d:0x0] from 18446744073709551615 #0 00000100:00100000:1.0:1775764499.375322:0:51218:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51218:1862022098379520:192.168.202.128@tcp:53:lhsmtool_posix.0 00000100:00100000:1.0:1775764499.375436:0:51218:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6f000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764499.386778:0:51218:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51218:1862022098379520:192.168.202.128@tcp:53:lhsmtool_posix.0 00000002:00000080:1.0:1775764499.386811:0:51218:0:(mdc_request.c:1758:mdc_ioc_fid2path()) path got [0x200000007:0x1:0x0] from 18446744073709551615 #0: f18.sanity-pcc 00000080:00200000:1.0:1775764499.386900:0:51218:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764499.386922:0:51218:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764499.386946:0:51218:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764499.386963:0:51218:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764499.386972:0:51218:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764499.386980:0:51218:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764499.386984:0:51218:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b17c 00000080:00010000:1.0:1775764499.386991:0:51218:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b17c from it ffffa60b01fb3bf8 00010000:00010000:1.0:1775764499.386995:0:51218:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764499.387004:0:51218:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764499.387016:0:51218:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764499.387114:0:51218:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764499.387123:0:51218:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764499.387129:0:51218:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764499.387135:0:51218:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764499.387330:0:51218:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764499.387335:0:51218:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764499.387337:0:51218:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b17c 00000080:00010000:1.0:1775764499.387340:0:51218:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b17c from it ffffa60b01fb3b38 00010000:00010000:1.0:1775764499.387342:0:51218:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764499.387348:0:51218:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00000080:00200000:1.0:1775764499.387354:0:51218:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 3 00000080:00200000:1.0:1775764499.387358:0:51218:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=. :VOLATILE:0000:42F0172C:fd=09, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), file ffff8b23a01a7c80, open_flags 280c2, mode 8180 opened 0 00000080:00200000:1.0:1775764499.387363:0:51218:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=. :VOLATILE:0000:42F0172C:fd=09, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=open|creat 00000002:00010000:1.0:1775764499.387382:0:51218:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: . :VOLATILE:0000:42F0172C:fd=09,[0x200003ab1:0x21:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0500303 00010000:00010000:1.0:1775764499.387394:0:51218:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764499.387440:0:51218:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764499.387449:0:51218:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764499.387491:0:51218:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4a00/0x73cfd86cfc17b17c lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304f4c6 expref: -99 pid: 51158 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764499.387521:0:51218:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4200/0x73cfd86cfc17b1d7 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 51218 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764499.387526:0:51218:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4200/0x73cfd86cfc17b1d7 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 51218 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764499.387534:0:51218:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764499.387541:0:51218:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd500 x1862022098379776/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 808/66448 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764499.387548:0:51218:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23bacb4200/0x73cfd86cfc17b1d7 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 51218 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764499.387563:0:51218:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51218:1862022098379776:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764499.432315:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118978176 00000100:00100000:1.0:1775764499.432333:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118978176 00000100:00100000:1.0:1775764499.558953:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118978304 00000100:00100000:1.0:1775764499.558966:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118978304 00000100:00100000:1.0:1775764499.559034:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 97 00000100:00100000:1.0:1775764499.559045:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42946:x1862022118978304:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764499.559067:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000006ca4d19b/0x73cfd86cfc17b1fa lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x109:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e0304f616 expref: -99 pid: 51218 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764499.559176:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764499.559184:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42946:x1862022118978304:12345-192.168.202.128@tcp:104: Request processed in 142us (370us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764499.559192:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 97 00000100:00100000:1.0:1775764499.564048:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764499.565437:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 98 00000100:00100000:1.0:1775764499.565448:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23864fe680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42309:x1862022118978432:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764499.565533:0:46139:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000901b77d0/0x73cfd86cfc17b201 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e0304f61d expref: -99 pid: 51218 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764499.565544:0:46139:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000901b77d0/0x73cfd86cfc17b201 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e0304f61d expref: -99 pid: 51218 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764499.565552:0:46139:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000901b77d0/0x73cfd86cfc17b201 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e0304f61d expref: -99 pid: 51218 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764499.565560:0:46139:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000901b77d0/0x73cfd86cfc17b201 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10080000020000 nid: local remote: 0x57b8ff0e0304f61d expref: -99 pid: 51218 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764499.565565:0:46139:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000901b77d0/0x73cfd86cfc17b201 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10080000020000 nid: local remote: 0x57b8ff0e0304f61d expref: -99 pid: 51218 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764499.565573:0:46139:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000901b77d0/0x73cfd86cfc17b201 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x80000020000 nid: local remote: 0x57b8ff0e0304f61d expref: -99 pid: 51218 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764499.565692:0:46139:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b23b9ceb400) 00000100:00100000:1.0:1775764499.565767:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23864fe680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42309:x1862022118978432:12345-192.168.202.128@tcp:105: Request processed in 321us (1734us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764499.565773:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 98 00000100:00100000:1.0:1775764499.715144:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098382720:192.168.202.128@tcp:16:lhsmtool_posix.0 00000100:00100000:1.0:1775764499.838960:0:46140:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764499.838982:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118978816 00000100:00100000:1.0:1775764499.838993:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118978816 00000100:00100000:1.0:1775764499.839080:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 99 00000100:00100000:1.0:1775764499.839091:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b238a397050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42326:x1862022118978816:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764499.839112:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9cea200/0x73cfd86cfc17b1e5 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x21:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e0304f5f3 expref: -99 pid: 51218 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764499.839208:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764499.839215:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b238a397050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42326:x1862022118978816:12345-192.168.202.128@tcp:104: Request processed in 126us (370us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764499.839224:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 99 00000100:00100000:1.0:1775764499.858817:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764499.910709:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764499.942953:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764501.013886:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118979584 00000100:00100000:1.0:1775764501.013897:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118979584 00000100:00100000:1.0:1775764501.013954:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 103 00000100:00100000:1.0:1775764501.013964:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dac380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42356:x1862022118979584:12345-192.168.202.128@tcp:107:hsm_cdtr.0 00000100:00100000:1.0:1775764501.014071:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dac380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42356:x1862022118979584:12345-192.168.202.128@tcp:107:hsm_cdtr.0 Request processed in 107us (304us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764501.014079:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 103 00000080:00200080:1.0:1775764501.015492:0:51310:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=c0086696 arg=7f2628000d20 00800000:00000080:1.0:1775764501.015517:0:51310:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=c0086696 len=4128 karg=ffff8b2384b0e000 uarg=0000000000000000 00000002:00000080:1.0:1775764501.015526:0:51310:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2384b87800: cmd=c0086696 len=4128 karg=ffff8b2384b0e000 uarg=0000000000000000 00000002:00000080:1.0:1775764501.015533:0:51310:0:(mdc_request.c:1738:mdc_ioc_fid2path()) path get [0x200003ab1:0x1d:0x0] from 18446744073709551615 #0 00000100:00100000:1.0:1775764501.015580:0:51310:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51310:1862022098384896:192.168.202.128@tcp:53:lhsmtool_posix.0 00000100:00100000:1.0:1775764501.015668:0:51310:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6f240 going to sleep for 11 seconds 00010000:00010000:1.0:1775764503.294846:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764505.337022:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1775764506.618347:0:51638:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764506.618367:0:51638:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764506.618400:0:51638:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764506.618683:0:51638:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b20f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 51638 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764506.618696:0:51638:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b20f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 51638 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764506.618706:0:51638:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b20f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 51638 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764506.618736:0:51638:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job lctl:7e6eda44-5508-487e-b17f-ff28ad9d3512:51638:1862022098386560:192.168.202.128@tcp:101:lctl.0 00000100:00100000:1.0:1775764506.618916:0:51638:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2383f7f840 going to sleep for 11 seconds 00000080:00200080:1.0:1775764506.841913:0:51094:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=401866d5 arg=1603980 00800000:00000080:1.0:1775764506.841933:0:51094:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=401866d5 len=24 karg=ffff8b23ab6b7700 uarg=0000000000000000 00000002:00000080:1.0:1775764506.841942:0:51094:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2384b87800: cmd=401866d5 len=24 karg=ffff8b23ab6b7700 uarg=0000000000000000 00000100:00100000:1.0:1775764506.841995:0:51094:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51094:1862022098386688:192.168.202.128@tcp:60:lhsmtool_posix.0 00000100:00100000:1.0:1775764506.842093:0:51094:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2383f7f840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764506.853268:0:51094:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51094:1862022098386688:192.168.202.128@tcp:60:lhsmtool_posix.0 00000080:00200008:1.0:1775764506.853370:0:51094:0:(file.c:447:ll_file_release()) START file fid:[0x200000002:0x2:0x0](ffff8b238990e3c8), flags 100000 00000002:00100000:1.0:1775764506.853446:0:51094:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fc700 x1862022098360192/t34359738638(34359738638) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/4688 e 0 to 0 dl 1775764510 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764506.853477:0:51094:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764506.853483:0:51094:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fc380 x1862022098386944/t0(0) o35->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:23/10 lens 392/752 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764506.853526:0:51094:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51094:1862022098386944:192.168.202.128@tcp:35:lhsmtool_posix.0 00000100:00100000:1.0:1775764506.853606:0:51094:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2383f7f840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764507.192193:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098387072:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764507.192290:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098387328:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764507.192316:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098387584:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764507.192340:0:2377:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1775764507.192345:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098387200:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764507.192363:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098387456:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764507.206697:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098387200:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764507.206717:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098387456:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764507.206729:0:2377:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b2384b87800: committing for last_committed 34359738401 gen 1 00000100:00100000:1.0:1775764507.206747:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098387328:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764507.206756:0:2377:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b238a3bf000: committing for last_committed 34359738398 gen 2 00000100:00100000:1.0:1775764507.206764:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098387584:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764507.216598:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098387072:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764512.318101:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2376:1862022098387840:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764512.318227:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098388096:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764512.318258:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2376:1862022098388352:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764512.318290:0:2376:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:1.0:1775764512.318297:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098387968:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764512.318317:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098388224:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764512.318336:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2376:1862022098388480:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764512.322958:0:2376:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2384b87800: committing for last_committed 34359738705 gen 1 00000100:00100000:1.0:1775764512.322975:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098387968:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764512.322990:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098388224:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764512.323000:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2376:1862022098388480:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764512.323021:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098388096:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764512.323032:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2376:1862022098388352:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764512.328829:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2376:1862022098387840:192.168.202.128@tcp:400:kworker.0 00010000:00010000:1.0:1775764516.604791:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1775764516.833102:0:51920:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764516.833121:0:51920:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764516.833134:0:51920:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b20f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f67f expref: -99 pid: 51638 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764516.833145:0:51920:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b20f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f67f expref: -99 pid: 51638 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764516.833152:0:51920:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764516.833159:0:51920:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764516.833163:0:51920:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b20f 00000080:00010000:1.0:1775764516.833167:0:51920:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b20f from it ffffa60b01573bb0 00010000:00010000:1.0:1775764516.833171:0:51920:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b20f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f67f expref: -99 pid: 51638 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764516.833179:0:51920:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b20f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f67f expref: -99 pid: 51638 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764516.833190:0:51920:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764516.833196:0:51920:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 100000 00000080:00200000:1.0:1775764516.833221:0:51920:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764516.833228:0:51920:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764516.833235:0:51920:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b20f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f67f expref: -99 pid: 51638 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764516.833243:0:51920:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b20f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f67f expref: -99 pid: 51638 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764516.833249:0:51920:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764516.833253:0:51920:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764516.833255:0:51920:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b20f 00000080:00010000:1.0:1775764516.833258:0:51920:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b20f from it ffffa60b01573bf8 00010000:00010000:1.0:1775764516.833261:0:51920:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b20f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f67f expref: -99 pid: 51638 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764516.833268:0:51920:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b20f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f67f expref: -99 pid: 51638 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764516.833275:0:51920:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764516.833282:0:51920:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=.lustre, flags=17 00000080:00200000:1.0:1775764516.833288:0:51920:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000002:0x1:0x0](ffff8b23899080c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764516.833300:0:51920:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=fid, dir=[0x200000002:0x1:0x0](ffff8b23899080c8), file ffff8b2383140a80, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1775764516.833306:0:51920:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=fid, dir=[0x200000002:0x1:0x0](ffff8b23899080c8), intent=open 00000002:00010000:1.0:1775764516.833323:0:51920:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: fid,[0x0:0x0:0x0]) in obj [0x200000002:0x1:0x0], intent: open flags 0100001 00010000:00010000:1.0:1775764516.833387:0:51920:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b21d lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 51920 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764516.833394:0:51920:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b21d lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 51920 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764516.833405:0:51920:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764516.833437:0:51920:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fce00 x1862022098388736/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/131984 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764516.833447:0:51920:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b21d lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 51920 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764516.833495:0:51920:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51920:1862022098388736:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764516.834077:0:51920:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23ab44ec00 going to sleep for 11 seconds 00000100:00100000:1.0:1775764517.201544:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118983680 00000100:00100000:1.0:1775764517.201557:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118983680 00000100:00100000:1.0:1775764517.201635:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 104 00000100:00100000:1.0:1775764517.201648:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dae300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42324:x1862022118983680:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764517.201679:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b9cebe00/0x73cfd86cfc17b216 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e0304f68d expref: -99 pid: 51659 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764517.201806:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764517.201814:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dae300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42324:x1862022118983680:12345-192.168.202.128@tcp:104: Request processed in 169us (565us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764517.201822:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 104 00000080:00200000:1.0:1775764517.275654:0:51934:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x22:0x0] updating i_size 0 i_blocks 1 00000020:00010000:1.0:1775764517.275668:0:51934:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027f478 (ffff8b23812ecb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764517.275675:0:51934:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027f478 (ffff8b23812ecb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764517.275682:0:51934:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e240b0 (ffff8b23812ec488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764517.275688:0:51934:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027f478 (ffff8b23812ecb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764517.275692:0:51934:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e240b0 (ffff8b23812ecb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764517.275758:0:51934:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x22:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1775764517.275941:0:51934:0:(file.c:2612:do_file_write_iter()) START file f19.sanity-pcc:[0x200003ab1:0x22:0x0], ppos: 0, count: 5 00000080:00200000:1.0:1775764517.275964:0:51934:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 5 bytes 00000080:00200000:1.0:1775764517.275973:0:51934:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 5 00000080:00200000:1.0:1775764517.276023:0:51934:0:(file.c:2058:ll_file_io_generic()) f19.sanity-pcc: write ppos: 0, bytes: 5 00000080:00200000:1.0:1775764517.276030:0:51934:0:(file.c:1815:ll_io_set_mirror()) f19.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764517.276039:0:51934:0:(cl_io.c:214:cl_io_rw_init()) header@00000000275ddf5c[0x110, 1, [0x200003ab1:0x22:0x0] hash] 00000020:00200000:1.0:1775764517.276041:0:51934:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 5) 0 0 00000080:00200000:1.0:1775764517.276047:0:51934:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x22:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764517.276068:0:51934:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1775764517.276081:0:51934:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 5) 5, 5 00020000:00200000:1.0:1775764517.276086:0:51934:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764517.276155:0:51934:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1775764517.276175:0:51934:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764517.276184:0:51934:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318930, osc lock ffff8b2382cf77c0, flags 0x0 00010000:00010000:1.0:1775764517.277082:0:51934:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b23831ed400 type 11 mode 2 res 9663677440/265 (0 4095) 00010000:00010000:1.0:1775764517.278545:0:51934:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x109:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764517.278569:0:51934:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x109:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764517.278585:0:51934:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x109:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764517.278625:0:51934:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:51934:1862022098389632:192.168.202.128@tcp:101:bash.0 00000100:00100000:1.0:1775764517.278763:0:51934:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23ab44e900 going to sleep for 11 seconds 00000100:00100000:1.0:1775764517.289544:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764517.306109:0:46140:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764517.306145:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118983936 00000100:00100000:1.0:1775764517.306160:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118983936 00000100:00100000:1.0:1775764517.306232:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 106 00000100:00100000:1.0:1775764517.306243:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23bad52450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42311:x1862022118983936:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764517.306264:0:46140:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f6e1 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764517.306281:0:46140:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f6e1 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764517.306291:0:46140:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f6e1 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764517.306318:0:46140:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e0304f6e1 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764517.306329:0:46140:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e0304f6e1 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764517.306340:0:46140:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000000000 nid: local remote: 0x57b8ff0e0304f6e1 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764517.306359:0:46140:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b23b8741400) 00000100:00100000:1.0:1775764517.306431:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23bad52450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42311:x1862022118983936:12345-192.168.202.128@tcp:105: Request processed in 189us (501us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764517.306439:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 106 00000100:00100000:1.0:1775764517.327256:0:51934:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:51934:1862022098389632:192.168.202.128@tcp:101:bash.0 00010000:00010000:1.0:1775764517.327313:0:51934:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e0304f6e1 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764517.327345:0:51934:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e0304f6e1 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1775764517.327361:0:51934:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000020000 nid: local remote: 0x57b8ff0e0304f6e1 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764517.327377:0:51934:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e0304f6e1 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764517.327387:0:51934:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e0304f6e1 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1775764517.327446:0:51934:0:(vvp_io.c:1286:vvp_io_write_start()) f19.sanity-pcc: write [0, 5) append 0 append lockpos 0 00000080:00200000:1.0:1775764517.327539:0:51934:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 5 bytes 00000080:00200000:1.0:1775764517.327647:0:51934:0:(rw26.c:858:ll_write_end()) pos 0, len 5, copied 5 00000080:00200000:1.0:1775764517.327667:0:51934:0:(rw26.c:916:ll_write_end()) page@00000000c8013407[3 000000002fca8c10 2 1 00000000e4726cd9] 00000080:00200000:1.0:1775764517.327671:0:51934:0:(rw26.c:916:ll_write_end()) vmpage @00000000a7323901 17ffffc0001001 3:0 ffff8b238a05f540 0 lru 00000080:00200000:1.0:1775764517.327692:0:51934:0:(rw26.c:916:ll_write_end()) osc-page@0000000034beaab6 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 00000000bf12bc0b 000000009c38a510 > 3< 0 0 > 4< 0 0 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764517.327694:0:51934:0:(rw26.c:916:ll_write_end()) end page@00000000c8013407 00000080:00200000:1.0:1775764517.327696:0:51934:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764517.327704:0:51934:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 5 prio 0 00000008:00010000:1.0:1775764517.327745:0:51934:0:(osc_cache.c:848:osc_extent_find()) ### extent: 00000000f6451404 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e0304f6e1 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1775764517.327781:0:51934:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b23899c4cb8, count 1, dirtied 1 00000080:00200000:1.0:1775764517.327789:0:51934:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 5 bytes, tot: 5 00000080:00200000:1.0:1775764517.327794:0:51934:0:(vvp_io.c:1353:vvp_io_write_start()) f19.sanity-pcc: write bytes 0, result: 5 00000020:00010000:1.0:1775764517.327935:0:51934:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb7308 (ffff8b2387cbaf38/0) at cl_lock_release():245 00000020:00010000:1.0:1775764517.327941:0:51934:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb7308 (ffff8b2387cbaf38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764517.327946:0:51934:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318930 (ffff8b2387cba998/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764517.327955:0:51934:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e0304f6e1 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764517.327968:0:51934:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e0304f6e1 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:1.0:1775764517.328002:0:51934:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb7308 (ffff8b2387cbaf38/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764517.328007:0:51934:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318930 (ffff8b2387cbaf38/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764517.328023:0:51934:0:(vvp_io.c:517:vvp_io_advance()) advancing 5 bytes 00000080:00200000:1.0:1775764517.328034:0:51934:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:1.0:1775764517.328093:0:51934:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x22:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764517.328099:0:51934:0:(file.c:2206:ll_file_io_generic()) f19.sanity-pcc: 2 io complete with rc: 0, result: 5, restart: 0 00000080:00200000:1.0:1775764517.328116:0:51934:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x22:0x0] updating i_size 5 i_blocks 1 00000080:00200000:1.0:1775764517.328126:0:51934:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 5 00000080:00200008:1.0:1775764517.328205:0:51934:0:(file.c:447:ll_file_release()) START file f19.sanity-pcc:[0x200003ab1:0x22:0x0](ffff8b23899c4b08), flags 100001 00000002:00100000:1.0:1775764517.328309:0:51934:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b2382dac700 x1862022098389248/t34359738707(34359738707) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764533 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764517.328324:0:51934:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764517.328330:0:51934:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fe300 x1862022098390016/t0(0) o35->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:23/10 lens 392/752 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764517.328369:0:51934:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:51934:1862022098390016:192.168.202.128@tcp:35:bash.0 00000100:00100000:1.0:1775764517.328582:0:51934:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23ab44e900 going to sleep for 11 seconds 00000100:00100000:1.0:1775764517.344253:0:51934:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:51934:1862022098390016:192.168.202.128@tcp:35:bash.0 00000002:00100000:1.0:1775764517.344333:0:51934:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b2382dac700 x1862022098389248/t34359738707(34359738707) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764533 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764517.452249:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2376:1862022098390144:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764517.452613:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2376:1862022098390400:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764517.452677:0:2376:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764517.452686:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098390272:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764517.479895:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098390272:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764517.479912:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2376:1862022098390144:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764517.479924:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2376:1862022098390400:192.168.202.128@tcp:400:kworker.0 00000080:00200000:1.0:1775764518.157330:0:51953:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x22:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764518.157352:0:51953:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:1.0:1775764518.157523:0:2376:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764518.157554:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098391296:192.168.202.128@tcp:1:lfs.0 00000100:00100000:1.0:1775764518.161735:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200000:1.0:1775764518.176663:0:46150:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x22:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764518.176760:0:46150:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x22:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1775764518.176867:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e0304f6e1 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764518.176876:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764518.176906:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e0304f6e1 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764518.176915:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000a1b263f6/0x73cfd86cfc17b232 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x109:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e0304f6e1 expref: -99 pid: 51934 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764518.176947:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764518.176949:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764518.176952:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764518.185941:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098391552:192.168.202.128@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764518.191052:0:51953:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b247 lrc: 2/0,0 mode: PW/PW res: [0x200003ab1:0x22:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x57b8ff0e0304f712 expref: -99 pid: 51953 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764518.191091:0:51953:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b247 lrc: 0/0,0 mode: --/PW res: [0x200003ab1:0x22:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x57b8ff0e0304f712 expref: -99 pid: 51953 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764518.191125:0:51953:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b2382dadc00 x1862022098391168/t34359738710(34359738710) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764534 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764518.191139:0:51953:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764518.191145:0:51953:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fe680 x1862022098391808/t0(0) o35->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:23/10 lens 504/752 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764518.191185:0:51953:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:51953:1862022098391808:192.168.202.128@tcp:35:lfs.0 00000100:00100000:1.0:1775764518.191401:0:51953:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23ab44e900 going to sleep for 11 seconds 00000100:00100000:1.0:1775764518.212568:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:1.0:1775764519.475441:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764519.475448:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f800/0x73cfd86cfc17b263 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x22:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304f782 expref: -99 pid: 51989 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764519.475713:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5f800 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764519.475718:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f800/0x73cfd86cfc17b263 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x22:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304f782 expref: -99 pid: 51989 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764519.475727:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f800/0x73cfd86cfc17b263 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x22:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304f782 expref: -99 pid: 51989 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764519.475736:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f800/0x73cfd86cfc17b263 lrc: 3/0,0 mode: PR/PR res: [0x200003ab1:0x22:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0304f782 expref: -99 pid: 51989 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764519.475758:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b255 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x22:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304f75f expref: -99 pid: 51970 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764519.475789:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f800/0x73cfd86cfc17b263 lrc: 2/0,0 mode: --/PR res: [0x200003ab1:0x22:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304f782 expref: -99 pid: 51989 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764519.475796:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764519.475820:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f800/0x73cfd86cfc17b263 lrc: 1/0,0 mode: --/PR res: [0x200003ab1:0x22:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304f782 expref: -99 pid: 51989 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764519.475828:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f800/0x73cfd86cfc17b263 lrc: 0/0,0 mode: --/PR res: [0x200003ab1:0x22:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304f782 expref: -99 pid: 51989 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764519.475848:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764519.475849:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764519.475851:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764519.475865:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098393856:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764519.576548:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118984704 00000100:00100000:1.0:1775764519.576568:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118984704 00000100:00100000:1.0:1775764519.576676:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 109 00000100:00100000:1.0:1775764519.576687:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daed80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42356:x1862022118984704:12345-192.168.202.128@tcp:107:hsm_cdtr.0 00000100:00100000:1.0:1775764519.576829:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daed80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42356:x1862022118984704:12345-192.168.202.128@tcp:107:hsm_cdtr.0 Request processed in 143us (585us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764519.576840:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 109 00000100:00100000:1.0:1775764519.826956:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764519.857174:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118984960 00000100:00100000:1.0:1775764519.857186:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118984960 00000100:00100000:1.0:1775764519.857247:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 111 00000100:00100000:1.0:1775764519.857258:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dadf80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:44302:x1862022118984960:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764519.857275:0:46140:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000006ca4d19b/0x73cfd86cfc17b2a9 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x10a:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e0304f83f expref: -99 pid: 51990 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764519.857288:0:46140:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000006ca4d19b/0x73cfd86cfc17b2a9 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x10a:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e0304f83f expref: -99 pid: 51990 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764519.857297:0:46140:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000006ca4d19b/0x73cfd86cfc17b2a9 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x10a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e0304f83f expref: -99 pid: 51990 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764519.857308:0:46140:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000006ca4d19b/0x73cfd86cfc17b2a9 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x10a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x57b8ff0e0304f83f expref: -99 pid: 51990 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764519.857318:0:46140:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000006ca4d19b/0x73cfd86cfc17b2a9 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x10a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x57b8ff0e0304f83f expref: -99 pid: 51990 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764519.857329:0:46140:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000006ca4d19b/0x73cfd86cfc17b2a9 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x10a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x57b8ff0e0304f83f expref: -99 pid: 51990 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764519.857360:0:46140:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b23b9ceba00) 00000100:00100000:1.0:1775764519.857577:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dadf80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:44302:x1862022118984960:12345-192.168.202.128@tcp:105: Request processed in 320us (530us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764519.857586:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 111 00000100:00100000:1.0:1775764520.183782:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764520.235945:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764520.251244:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118985600 00000100:00100000:1.0:1775764520.251256:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118985600 00000100:00100000:1.0:1775764520.251351:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 114 00000100:00100000:1.0:1775764520.251361:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daed80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42326:x1862022118985600:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764520.251396:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e800/0x73cfd86cfc17b294 lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x24:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0304f823 expref: -99 pid: 51990 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764520.251688:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764520.251694:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daed80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42326:x1862022118985600:12345-192.168.202.128@tcp:104: Request processed in 336us (580us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764520.251702:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 114 00000100:00100000:1.0:1775764520.266654:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764520.322899:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764520.345119:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118986240 00000100:00100000:1.0:1775764520.345131:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118986240 00000100:00100000:1.0:1775764520.345185:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 117 00000100:00100000:1.0:1775764520.345194:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dafb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42327:x1862022118986240:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764520.345211:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9cea600/0x73cfd86cfc17b271 lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x22:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0304f7ac expref: -99 pid: 51989 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764520.345293:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764520.345298:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dafb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42327:x1862022118986240:12345-192.168.202.128@tcp:104: Request processed in 106us (336us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764520.345306:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 117 00000100:00100000:1.0:1775764520.755726:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764520.755765:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118986624 00000100:00100000:1.0:1775764520.755774:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118986624 00000100:00100000:1.0:1775764520.755821:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 118 00000100:00100000:1.0:1775764520.755830:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23864fca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:44493:x1862022118986624:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764520.755857:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f200/0x73cfd86cfc17b2c5 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x22:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0304f8bd expref: -99 pid: 51997 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764520.755933:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764520.755940:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23864fca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:44493:x1862022118986624:12345-192.168.202.128@tcp:104: Request processed in 111us (223us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764520.755947:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 118 00010000:00010000:1.0:1775764521.727569:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764522.553719:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2376:1862022098401792:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764522.553768:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098402048:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764522.553793:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2376:1862022098402304:192.168.202.128@tcp:400:kworker.0 00010000:00010000:1.0:1775764523.769752:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764525.815656:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764526.971683:0:51920:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2384b87800: committing for last_committed 34359738747 gen 1 00000100:00100000:1.0:1775764526.971722:0:51920:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23b9ea4000 x1862022098401536/t34359738747(34359738747) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764536 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764526.971735:0:51920:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51920:1862022098402560:192.168.202.128@tcp:60:lhsmtool_posix.0 00000080:00200008:1.0:1775764526.971837:0:51920:0:(file.c:447:ll_file_release()) START file fid:[0x200000002:0x2:0x0](ffff8b238990e3c8), flags 100000 00000002:00100000:1.0:1775764526.971912:0:51920:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fce00 x1862022098388736/t34359738706(34359738706) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/4688 e 0 to 0 dl 1775764532 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764526.971925:0:51920:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764526.971930:0:51920:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd180 x1862022098402816/t0(0) o35->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:23/10 lens 392/752 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764526.971955:0:51920:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51920:1862022098402816:192.168.202.128@tcp:35:lhsmtool_posix.0 00000100:00100000:1.0:1775764526.972061:0:51920:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23831a2780 going to sleep for 11 seconds 00000100:00100000:1.0:1775764526.977725:0:51920:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:51920:1862022098402816:192.168.202.128@tcp:35:lhsmtool_posix.0 00000002:00100000:1.0:1775764526.977762:0:51920:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fce00 x1862022098388736/t34359738706(34359738706) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/4688 e 0 to 0 dl 1775764532 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1775764526.977828:0:51920:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 100000 00000080:00200008:1.0:1775764526.980604:0:51920:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 100000 00000080:00200008:1.0:1775764527.280832:0:52450:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764527.280851:0:52450:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764527.280885:0:52450:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764527.280911:0:52450:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e600/0x73cfd86cfc17b2be lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f8a1 expref: -99 pid: 51997 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764527.280931:0:52450:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e600/0x73cfd86cfc17b2be lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f8a1 expref: -99 pid: 51997 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764527.280940:0:52450:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764527.280955:0:52450:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764527.280960:0:52450:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b2be 00000080:00010000:1.0:1775764527.280969:0:52450:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b2be from it ffffa60b01d8bcb8 00010000:00010000:1.0:1775764527.280974:0:52450:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e600/0x73cfd86cfc17b2be lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f8a1 expref: -99 pid: 51997 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764527.280984:0:52450:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e600/0x73cfd86cfc17b2be lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f8a1 expref: -99 pid: 51997 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200008:1.0:1775764527.283587:0:52450:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764527.283625:0:52450:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:1.0:1775764527.283658:0:52450:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764527.283727:0:52450:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741a00/0x73cfd86cfc17b2d3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 52450 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764527.283737:0:52450:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741a00/0x73cfd86cfc17b2d3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 52450 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764527.283748:0:52450:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741a00/0x73cfd86cfc17b2d3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 52450 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764527.283777:0:52450:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lsof:82781e18-e0e3-4f9f-9a2b-041bf0344414:52450:1862022098402944:192.168.202.128@tcp:101:lsof.0 00000100:00100000:1.0:1775764527.283941:0:52450:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23831a2780 going to sleep for 11 seconds 00000100:00100000:1.0:1775764527.291705:0:52450:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lsof:82781e18-e0e3-4f9f-9a2b-041bf0344414:52450:1862022098402944:192.168.202.128@tcp:101:lsof.0 00010000:00010000:1.0:1775764527.291734:0:52450:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741a00/0x73cfd86cfc17b2d3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f8e7 expref: -99 pid: 52450 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764527.291746:0:52450:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741a00/0x73cfd86cfc17b2d3 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f8e7 expref: -99 pid: 52450 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764527.291757:0:52450:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741a00/0x73cfd86cfc17b2d3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f8e7 expref: -99 pid: 52450 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764527.291768:0:52450:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741a00/0x73cfd86cfc17b2d3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f8e7 expref: -99 pid: 52450 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764527.291786:0:52450:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741a00/0x73cfd86cfc17b2d3 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f8e7 expref: -99 pid: 52450 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764527.291794:0:52450:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741a00/0x73cfd86cfc17b2d3 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f8e7 expref: -99 pid: 52450 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764527.291801:0:52450:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741a00/0x73cfd86cfc17b2d3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f8e7 expref: -99 pid: 52450 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764527.291811:0:52450:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864fd180 x1862022098402944/t0(0) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764543 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lsof.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764527.291824:0:52450:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741a00/0x73cfd86cfc17b2d3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f8e7 expref: -99 pid: 52450 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764527.291837:0:52450:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2387c07800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1775764527.291843:0:52450:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764527.291875:0:52450:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 8704 00000080:00010000:1.0:1775764527.291881:0:52450:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17b2d3 00000080:00010000:1.0:1775764527.291886:0:52450:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:1.0:1775764527.291889:0:52450:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b2d3 from it ffffa60b01d8bcb8 00010000:00010000:1.0:1775764527.291892:0:52450:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741a00/0x73cfd86cfc17b2d3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f8e7 expref: -99 pid: 52450 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764527.291900:0:52450:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741a00/0x73cfd86cfc17b2d3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f8e7 expref: -99 pid: 52450 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764530.935759:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764532.809916:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098403712:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764532.809939:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098404224:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764532.809949:0:2377:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2384b87800: committing for last_committed 34359738748 gen 1 00000100:00100000:1.0:1775764532.809953:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098403840:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764532.815085:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098403968:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764532.825416:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098404096:192.168.202.128@tcp:400:kworker.0 00010000:00010000:1.0:1775764532.984086:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1775764535.519285:0:52709:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764535.519299:0:52709:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764535.519319:0:52709:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764535.519338:0:52709:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e600/0x73cfd86cfc17b2be lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f8a1 expref: -99 pid: 51997 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764535.519364:0:52709:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e600/0x73cfd86cfc17b2be lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f8a1 expref: -99 pid: 51997 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764535.519372:0:52709:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764535.519380:0:52709:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764535.519384:0:52709:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b2be 00000080:00010000:1.0:1775764535.519391:0:52709:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b2be from it ffffa60b01e33cb8 00010000:00010000:1.0:1775764535.519395:0:52709:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e600/0x73cfd86cfc17b2be lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f8a1 expref: -99 pid: 51997 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764535.519403:0:52709:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e600/0x73cfd86cfc17b2be lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f8a1 expref: -99 pid: 51997 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764535.537157:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764535.537166:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fc00/0x73cfd86cfc17b2da lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304f903 expref: -99 pid: 52710 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764535.537175:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5fc00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764535.537180:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fc00/0x73cfd86cfc17b2da lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304f903 expref: -99 pid: 52710 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764535.537189:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fc00/0x73cfd86cfc17b2da lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304f903 expref: -99 pid: 52710 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764535.537198:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fc00/0x73cfd86cfc17b2da lrc: 3/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0304f903 expref: -99 pid: 52710 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764535.537218:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8606000/0x73cfd86cfc17b0cd lrc: 3/1,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f273 expref: -99 pid: 51094 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764535.537741:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fc00/0x73cfd86cfc17b2da lrc: 2/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304f903 expref: -99 pid: 52710 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764535.537751:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764535.537783:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fc00/0x73cfd86cfc17b2da lrc: 1/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304f903 expref: -99 pid: 52710 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764535.537790:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fc00/0x73cfd86cfc17b2da lrc: 0/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304f903 expref: -99 pid: 52710 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764535.537810:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764535.537812:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764535.537813:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764535.537835:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098404608:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764535.543554:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098404608:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764535.692123:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764535.717622:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764535.718091:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 120 00000100:00100000:1.0:1775764535.718104:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23864fd180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42946:x1862022118989696:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764535.718126:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 0000000078667c02/0x73cfd86cfc17b2e8 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x10a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e0304f934 expref: -99 pid: 52724 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764535.718215:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764535.718222:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23864fd180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42946:x1862022118989696:12345-192.168.202.128@tcp:104: Request processed in 122us (612us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764535.718229:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 120 00000100:00100000:1.0:1775764535.725750:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764535.826718:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764535.856620:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764535.875192:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764535.876278:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 124 00000100:00100000:1.0:1775764535.876290:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23864fd500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42328:x1862022118990208:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764535.876309:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e600/0x73cfd86cfc17b30b lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x26:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0304f97a expref: -99 pid: 52730 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764535.876555:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764535.876563:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23864fd500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42328:x1862022118990208:12345-192.168.202.128@tcp:104: Request processed in 274us (1382us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764535.876571:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 124 00010000:00010000:1.0:1775764536.056325:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764538.113443:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1775764538.370187:0:52778:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764538.370201:0:52778:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764538.370928:0:52778:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764538.370948:0:52778:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50400/0x73cfd86cfc17b2f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f949 expref: -99 pid: 52730 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764538.370963:0:52778:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50400/0x73cfd86cfc17b2f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f949 expref: -99 pid: 52730 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764538.370971:0:52778:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764538.370981:0:52778:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764538.370984:0:52778:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b2f6 00000080:00010000:1.0:1775764538.370991:0:52778:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b2f6 from it ffffa60b01efbcb8 00010000:00010000:1.0:1775764538.370994:0:52778:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50400/0x73cfd86cfc17b2f6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f949 expref: -99 pid: 52730 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764538.371004:0:52778:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50400/0x73cfd86cfc17b2f6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f949 expref: -99 pid: 52730 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764538.371039:0:52778:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764538.371047:0:52778:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764538.371053:0:52778:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50400/0x73cfd86cfc17b2f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f949 expref: -99 pid: 52730 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764538.371061:0:52778:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50400/0x73cfd86cfc17b2f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f949 expref: -99 pid: 52730 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764538.371067:0:52778:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764538.371071:0:52778:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764538.371073:0:52778:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b2f6 00000080:00010000:1.0:1775764538.371076:0:52778:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b2f6 from it ffffa60b01efbb58 00010000:00010000:1.0:1775764538.371079:0:52778:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50400/0x73cfd86cfc17b2f6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f949 expref: -99 pid: 52730 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764538.371086:0:52778:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50400/0x73cfd86cfc17b2f6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f949 expref: -99 pid: 52730 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764538.371094:0:52778:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764538.371291:0:52778:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=f20.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags=8 00000080:00200000:1.0:1775764538.371298:0:52778:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f20.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=getattr 00000002:00010000:1.0:1775764538.371313:0:52778:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f20.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764538.371369:0:52778:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b319 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 52778 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764538.371378:0:52778:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b319 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 52778 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764538.371387:0:52778:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b319 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 52778 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764538.371422:0:52778:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:52778:1862022098409344:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764538.371964:0:52778:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23831a2540 going to sleep for 11 seconds 00010000:00010000:1.0:1775764538.433051:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764538.433060:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f000/0x73cfd86cfc17b320 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304f9e3 expref: -99 pid: 52778 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764538.433070:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5f000 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764538.433076:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f000/0x73cfd86cfc17b320 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304f9e3 expref: -99 pid: 52778 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764538.433085:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f000/0x73cfd86cfc17b320 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304f9e3 expref: -99 pid: 52778 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764538.433094:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f000/0x73cfd86cfc17b320 lrc: 3/0,0 mode: PR/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0304f9e3 expref: -99 pid: 52778 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764538.433118:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b319 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304f9d5 expref: -99 pid: 52778 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764538.433172:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f000/0x73cfd86cfc17b320 lrc: 2/0,0 mode: --/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304f9e3 expref: -99 pid: 52778 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764538.433181:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764538.433227:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f000/0x73cfd86cfc17b320 lrc: 1/0,0 mode: --/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304f9e3 expref: -99 pid: 52778 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764538.433235:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f000/0x73cfd86cfc17b320 lrc: 0/0,0 mode: --/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304f9e3 expref: -99 pid: 52778 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764538.433253:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764538.433255:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764538.433257:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764538.433271:0:2377:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764538.433289:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098409728:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764538.462170:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098409728:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764538.466899:0:52778:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:52778:1862022098409984:192.168.202.128@tcp:103:lfs.0 00010000:00010000:1.0:1775764538.466926:0:52778:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b319 lrc: 0/0,0 mode: --/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e0304f9d5 expref: -99 pid: 52778 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764538.466943:0:52778:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9cebe00/0x73cfd86cfc17b312 lrc: 0/0,0 mode: --/CR res: [0x200003ab1:0x26:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e0304f98f expref: -99 pid: 52730 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1775764538.466970:0:52778:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 58, max 7 00000100:00100000:1.0:1775764538.466979:0:52778:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864f1c00 x1862022098410112/t0(0) o58->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764538.467004:0:52778:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:52778:1862022098410112:192.168.202.128@tcp:58:lfs.0 00000100:00100000:1.0:1775764538.467172:0:52778:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23831a2540 going to sleep for 11 seconds 00000100:00100000:1.0:1775764538.477411:0:52778:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:52778:1862022098410112:192.168.202.128@tcp:58:lfs.0 00000080:00010000:1.0:1775764538.477495:0:52778:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab1:0x26:0x0](ffff8b238990db88) 00000002:00010000:1.0:1775764538.477509:0:52778:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x26:0x0]) in obj [0x200003ab1:0x26:0x0], intent: layout flags 0 00010000:00010000:1.0:1775764538.477563:0:52778:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740600/0x73cfd86cfc17b327 lrc: 3/1,0 mode: --/CR res: [0x200003ab1:0x26:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 52778 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764538.477577:0:52778:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740600/0x73cfd86cfc17b327 lrc: 3/1,0 mode: --/CR res: [0x200003ab1:0x26:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 52778 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764538.477589:0:52778:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740600/0x73cfd86cfc17b327 lrc: 3/1,0 mode: --/CR res: [0x200003ab1:0x26:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 52778 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764538.477612:0:52778:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:52778:1862022098410240:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764538.477675:0:52778:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23831a2540 going to sleep for 11 seconds 00000100:00100000:1.0:1775764538.487276:0:52778:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:52778:1862022098410240:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764538.487307:0:52778:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740600/0x73cfd86cfc17b327 lrc: 4/1,0 mode: --/CR res: [0x200003ab1:0x26:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f9f8 expref: -99 pid: 52778 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764538.487320:0:52778:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740600/0x73cfd86cfc17b327 lrc: 4/1,0 mode: --/CR res: [0x200003ab1:0x26:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f9f8 expref: -99 pid: 52778 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764538.717219:0:46139:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764538.717249:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118991360 00000100:00100000:1.0:1775764538.717263:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118991360 00000100:00100000:1.0:1775764538.717366:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 125 00000100:00100000:1.0:1775764538.717377:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23b9ff0450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42356:x1862022118991360:12345-192.168.202.128@tcp:107:hsm_cdtr.0 00000100:00100000:1.0:1775764538.717526:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23b9ff0450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42356:x1862022118991360:12345-192.168.202.128@tcp:107:hsm_cdtr.0 Request processed in 150us (932us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764538.717535:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 125 00000100:00100000:1.0:1775764538.980730:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118991488 00000100:00100000:1.0:1775764538.980745:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118991488 00000100:00100000:1.0:1775764538.980815:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 126 00000100:00100000:1.0:1775764538.980826:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dafb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43212:x1862022118991488:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764538.980849:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000557ec74c/0x73cfd86cfc17b358 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x10b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e0304fa84 expref: -99 pid: 52779 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764538.980952:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764538.980959:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dafb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43212:x1862022118991488:12345-192.168.202.128@tcp:104: Request processed in 134us (342us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764538.980966:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 126 00000100:00100000:1.0:1775764538.994551:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:1.0:1775764539.131255:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1775764539.304733:0:52779:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x28:0x0] updating i_size 5 i_blocks 11 00000080:00200000:1.0:1775764539.304780:0:52779:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x28:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:1.0:1775764539.304810:0:52779:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab1:0x28:0x0](ffff8b2390ec0908) cmd=c03066b0 arg=7fe623e12ac0 00000080:00200008:1.0:1775764539.304823:0:52779:0:(llite_lib.c:2274:ll_setattr_raw()) START file . :VOLATILE:0000:5C99EB77:fd=09:[0x200003ab1:0x28:0x0](ffff8b2390ec0908) current size 5, valid attrs 1f0, mode 0, uid 0, gid 0, new size 0, atime 1775764535.000000000, mtime 1775764535.000000000, ctime 1775764535.000000000 00010000:00010000:1.0:1775764539.304866:0:52779:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e800/0x73cfd86cfc17b351 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x28:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e0304fa7d expref: -99 pid: 52779 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764539.304895:0:52779:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380031800/0x73cfd86cfc17b34a lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x28:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fa6f expref: -99 pid: 52779 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764539.304955:0:52779:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e800/0x73cfd86cfc17b351 lrc: 1/0,0 mode: --/PR res: [0x200003ab1:0x28:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e0304fa7d expref: -99 pid: 52779 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764539.304962:0:52779:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764539.304965:0:52779:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e800/0x73cfd86cfc17b351 lrc: 0/0,0 mode: --/PR res: [0x200003ab1:0x28:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e0304fa7d expref: -99 pid: 52779 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000020:00100000:1.0:1775764539.304994:0:52779:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1775764539.305003:0:52779:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fce00 x1862022098414080/t0(0) o36->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764539.305033:0:52779:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:52779:1862022098414080:192.168.202.128@tcp:36:lhsmtool_posix.0 00000100:00100000:1.0:1775764539.305141:0:52779:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23812ed900 going to sleep for 11 seconds 00000100:00100000:1.0:1775764539.333049:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118992128 00000100:00100000:1.0:1775764539.333059:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118992128 00000100:00100000:1.0:1775764539.333113:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 128 00000100:00100000:1.0:1775764539.333123:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dac000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42325:x1862022118992128:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764539.333141:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e000/0x73cfd86cfc17b343 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x28:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e0304fa61 expref: -99 pid: 52779 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764539.333218:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764539.333225:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dac000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42325:x1862022118992128:12345-192.168.202.128@tcp:104: Request processed in 104us (362us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764539.333231:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 128 00000100:00100000:1.0:1775764539.383228:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200080:1.0:1775764540.446271:0:52864:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=c0086696 arg=7fe61c000d20 00800000:00000080:1.0:1775764540.446295:0:52864:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=c0086696 len=4128 karg=ffff8b2384b0c000 uarg=0000000000000000 00000002:00000080:1.0:1775764540.446314:0:52864:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2384b87800: cmd=c0086696 len=4128 karg=ffff8b2384b0c000 uarg=0000000000000000 00000002:00000080:1.0:1775764540.446321:0:52864:0:(mdc_request.c:1738:mdc_ioc_fid2path()) path get [0x200003ab1:0x26:0x0] from 18446744073709551615 #0 00000100:00100000:1.0:1775764540.446369:0:52864:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:52864:1862022098416256:192.168.202.128@tcp:53:lhsmtool_posix.0 00000100:00100000:1.0:1775764540.446580:0:52864:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23812ed240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764540.508688:0:52864:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:52864:1862022098416512:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:1.0:1775764540.508731:0:52864:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f200/0x73cfd86cfc17b366 lrc: 4/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fad1 expref: -99 pid: 52864 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764540.508752:0:52864:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f200/0x73cfd86cfc17b366 lrc: 4/1,0 mode: --/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fad1 expref: -99 pid: 52864 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764540.508761:0:52864:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f200/0x73cfd86cfc17b366 lrc: 4/1,0 mode: --/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fad1 expref: -99 pid: 52864 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764540.508775:0:52864:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f200/0x73cfd86cfc17b366 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fad1 expref: -99 pid: 52864 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764540.508787:0:52864:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f200/0x73cfd86cfc17b366 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fad1 expref: -99 pid: 52864 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764540.508803:0:52864:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f200/0x73cfd86cfc17b366 lrc: 5/2,0 mode: PR/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fad1 expref: -99 pid: 52864 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764540.508812:0:52864:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f200/0x73cfd86cfc17b366 lrc: 5/2,0 mode: PR/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fad1 expref: -99 pid: 52864 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764540.508818:0:52864:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f200/0x73cfd86cfc17b366 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fad1 expref: -99 pid: 52864 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764540.508829:0:52864:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23864f0380 x1862022098416512/t34359738781(34359738781) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/664 e 0 to 0 dl 1775764556 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764540.508849:0:52864:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864f0380 x1862022098416512/t34359738781(34359738781) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/664 e 0 to 0 dl 1775764556 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764540.508863:0:52864:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f200/0x73cfd86cfc17b366 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fad1 expref: -99 pid: 52864 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764540.508878:0:52864:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f200/0x73cfd86cfc17b366 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fad1 expref: -99 pid: 52864 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764540.508889:0:52864:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 4 res 8589949617/38 (0 0) 00000080:00200000:1.0:1775764540.508895:0:52864:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000002:0x2:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764540.509293:0:52864:0:(namei.c:147:ll_iget()) got inode: [0x200003ab1:0x26:0x0](ffff8b238990db88): rc = 0 00000080:00200000:1.0:1775764540.509326:0:52864:0:(file.c:6766:ll_layout_conf()) [0x200003ab1:0x26:0x0]: layout version change: 2 -> 2 00000080:00010000:1.0:1775764540.509339:0:52864:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x26:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17b366 00000080:00010000:1.0:1775764540.509351:0:52864:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x26:0x0](ffff8b238990db88) 00000080:00200008:1.0:1775764540.509367:0:52864:0:(file.c:1057:ll_file_open()) START file 0x200003ab1:0x26:0x0:[0x200003ab1:0x26:0x0](ffff8b238990db88), flags 1504000 00000002:00100000:1.0:1775764540.509390:0:52864:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864f0380 x1862022098416512/t34359738781(34359738781) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764556 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1775764540.509441:0:52864:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b366 from it ffff8b2389ee3000 00010000:00010000:1.0:1775764540.509448:0:52864:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f200/0x73cfd86cfc17b366 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0304fad1 expref: -99 pid: 52864 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764540.509488:0:52864:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5f200/0x73cfd86cfc17b366 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x26:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fad1 expref: -99 pid: 52864 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764540.509513:0:52864:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab1:0x26:0x0](ffff8b238990db88), inode mode 81a4 mask 44 00000080:00200080:1.0:1775764540.509925:0:52864:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=400866d7 arg=7fe61c000b78 00800000:00000080:1.0:1775764540.509938:0:52864:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=401866d8 len=64 karg=ffffa60b01e0bcf0 uarg=0000000000000000 00000002:00000080:1.0:1775764540.509945:0:52864:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2384b87800: cmd=401866d8 len=64 karg=ffffa60b01e0bcf0 uarg=0000000000000000 00000020:00100000:1.0:1775764540.509994:0:52864:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 57, max 7 00000100:00100000:1.0:1775764540.510002:0:52864:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd500 x1862022098416768/t0(0) o57->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764540.510028:0:52864:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:52864:1862022098416768:192.168.202.128@tcp:57:lhsmtool_posix.0 00000100:00100000:1.0:1775764540.511658:0:52864:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23812ed240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764540.535266:0:52864:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:52864:1862022098416768:192.168.202.128@tcp:57:lhsmtool_posix.0 00000080:00200008:1.0:1775764540.535324:0:52864:0:(file.c:447:ll_file_release()) START file 0x200003ab1:0x26:0x0:[0x200003ab1:0x26:0x0](ffff8b238990db88), flags 1504000 00000002:00100000:1.0:1775764540.535395:0:52864:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864f0380 x1862022098416512/t34359738781(34359738781) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764556 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764540.535407:0:52864:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764540.535413:0:52864:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fdc00 x1862022098417024/t0(0) o35->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:23/10 lens 392/752 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764540.535440:0:52864:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:52864:1862022098417024:192.168.202.128@tcp:35:lhsmtool_posix.0 00000100:00100000:1.0:1775764540.535568:0:52864:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23812ed240 going to sleep for 11 seconds 00010000:00010000:1.0:1775764541.180607:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764543.037524:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098417280:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764543.037632:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098417536:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764543.037674:0:2377:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:1.0:1775764543.037680:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098417152:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764543.037708:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098417408:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764543.037729:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098417664:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764543.057427:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098417152:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764543.061931:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098417664:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764543.061966:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098417536:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764543.076731:0:2377:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b2384b87800: committing for last_committed 34359738404 gen 2 00000100:00100000:1.0:1775764543.076774:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098417408:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764543.076792:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098417280:192.168.202.128@tcp:400:kworker.0 00010000:00010000:1.0:1775764545.337661:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764548.155899:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098418432:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764548.156020:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098418688:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764548.156052:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098418944:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764548.156080:0:2377:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1775764548.156085:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098418560:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764548.156102:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098418816:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764548.169499:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098418560:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764548.169529:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098418688:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764548.169543:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098418944:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764548.189844:0:2377:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b2384b87800: committing for last_committed 34359738413 gen 1 00000100:00100000:1.0:1775764548.189888:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098418816:192.168.202.128@tcp:400:kworker.0 00000080:00200000:1.0:1775764548.668381:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x22:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764548.668398:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x22:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1775764553.912158:0:53479:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764553.912176:0:53479:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764553.912900:0:53479:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764553.912925:0:53479:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fe00/0x73cfd86cfc17b36d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304faed expref: -99 pid: 53203 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764553.912944:0:53479:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fe00/0x73cfd86cfc17b36d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304faed expref: -99 pid: 53203 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764553.912954:0:53479:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764553.912966:0:53479:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764553.912970:0:53479:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b36d 00000080:00010000:1.0:1775764553.912979:0:53479:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b36d from it ffffa60b01de7cb8 00010000:00010000:1.0:1775764553.912984:0:53479:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fe00/0x73cfd86cfc17b36d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304faed expref: -99 pid: 53203 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764553.912994:0:53479:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fe00/0x73cfd86cfc17b36d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304faed expref: -99 pid: 53203 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200008:1.0:1775764553.914940:0:53479:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764553.914952:0:53479:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764553.914977:0:53479:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764553.914997:0:53479:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fe00/0x73cfd86cfc17b36d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304faed expref: -99 pid: 53203 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764553.915014:0:53479:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fe00/0x73cfd86cfc17b36d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304faed expref: -99 pid: 53203 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764553.915022:0:53479:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764553.915031:0:53479:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764553.915035:0:53479:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b36d 00000080:00010000:1.0:1775764553.915042:0:53479:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b36d from it ffffa60b01de7cb8 00010000:00010000:1.0:1775764553.915046:0:53479:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fe00/0x73cfd86cfc17b36d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304faed expref: -99 pid: 53203 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764553.915055:0:53479:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fe00/0x73cfd86cfc17b36d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304faed expref: -99 pid: 53203 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764553.915210:0:53479:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764553.915218:0:53479:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764553.915225:0:53479:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fe00/0x73cfd86cfc17b36d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304faed expref: -99 pid: 53203 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764553.915234:0:53479:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fe00/0x73cfd86cfc17b36d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304faed expref: -99 pid: 53203 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764553.915240:0:53479:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764553.915244:0:53479:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764553.915246:0:53479:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b36d 00000080:00010000:1.0:1775764553.915249:0:53479:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b36d from it ffffa60b01de7bb0 00010000:00010000:1.0:1775764553.915253:0:53479:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fe00/0x73cfd86cfc17b36d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304faed expref: -99 pid: 53203 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764553.915259:0:53479:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fe00/0x73cfd86cfc17b36d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304faed expref: -99 pid: 53203 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764553.915269:0:53479:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764553.915277:0:53479:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 100000 00000100:00100000:1.0:1775764554.162968:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118995968 00000100:00100000:1.0:1775764554.162979:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118995968 00000100:00100000:1.0:1775764554.163054:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 133 00000100:00100000:1.0:1775764554.163072:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dadc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42324:x1862022118995968:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764554.163095:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030a00/0x73cfd86cfc17b374 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e0304fafb expref: -99 pid: 53220 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764554.163214:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764554.163222:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dadc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42324:x1862022118995968:12345-192.168.202.128@tcp:104: Request processed in 159us (398us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764554.163230:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 133 00000100:00100000:1.0:1775764554.205098:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118996096 00000100:00100000:1.0:1775764554.205109:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118996096 00000100:00100000:1.0:1775764554.205168:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 134 00000100:00100000:1.0:1775764554.205186:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43212:x1862022118996096:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764554.205205:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b389 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x10b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e0304fb48 expref: -99 pid: 53494 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764554.205300:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764554.205307:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43212:x1862022118996096:12345-192.168.202.128@tcp:104: Request processed in 132us (335us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764554.205314:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 134 00000100:00100000:1.0:1775764554.241173:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200000:1.0:1775764554.788893:0:53511:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764554.788930:0:53511:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764554.788954:0:53511:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764554.788976:0:53511:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764554.788986:0:53511:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764554.788998:0:53511:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764554.789003:0:53511:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b397 00000080:00010000:1.0:1775764554.789011:0:53511:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b397 from it ffffa60b01e7bbb0 00010000:00010000:1.0:1775764554.789015:0:53511:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764554.789025:0:53511:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764554.789046:0:53511:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764554.789056:0:53511:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 304000 00000080:00200080:1.0:1775764554.789091:0:53511:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=901066fc arg=7ffdd8456790 00000100:00100000:1.0:1775764554.789201:0:53511:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:53511:1862022098422528:192.168.202.128@tcp:34:lfs.0 00000100:00100000:1.0:1775764554.789388:0:53511:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2382bb59c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764554.798408:0:53511:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:53511:1862022098422528:192.168.202.128@tcp:34:lfs.0 00000080:00200008:1.0:1775764554.798520:0:53511:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 304000 00000080:00200008:1.0:1775764554.975204:0:53518:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764554.975222:0:53518:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764554.975248:0:53518:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764554.975274:0:53518:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764554.975295:0:53518:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764554.975304:0:53518:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764554.975313:0:53518:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764554.975316:0:53518:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b397 00000080:00010000:1.0:1775764554.975324:0:53518:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b397 from it ffffa60b01e5bcb8 00010000:00010000:1.0:1775764554.975328:0:53518:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764554.975337:0:53518:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764554.975370:0:53518:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764554.975376:0:53518:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764554.975381:0:53518:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764554.975389:0:53518:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764554.975400:0:53518:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764554.975405:0:53518:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764554.975407:0:53518:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b397 00000080:00010000:1.0:1775764554.975410:0:53518:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b397 from it ffffa60b01e5bb58 00010000:00010000:1.0:1775764554.975414:0:53518:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764554.975420:0:53518:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764554.975429:0:53518:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764554.975449:0:53518:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=f21a.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags=8 00000080:00200000:1.0:1775764554.975958:0:53518:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f21a.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=getattr 00000002:00010000:1.0:1775764554.975978:0:53518:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f21a.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764554.976050:0:53518:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 53518 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764554.976059:0:53518:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 53518 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764554.976073:0:53518:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 53518 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764554.976109:0:53518:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:53518:1862022098423040:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764554.976266:0:53518:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2382bb59c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764554.997058:0:53518:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:53518:1862022098423040:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764554.997103:0:53518:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764554.997125:0:53518:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 4/1,0 mode: --/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764554.997134:0:53518:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 4/1,0 mode: --/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764554.997149:0:53518:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764554.997161:0:53518:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764554.997182:0:53518:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 5/2,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764554.997190:0:53518:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 5/2,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764554.997198:0:53518:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764554.997212:0:53518:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864ff480 x1862022098423040/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 584/840 e 0 to 0 dl 1775764570 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764554.997224:0:53518:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764554.997245:0:53518:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764554.997258:0:53518:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 4 res 8589949617/41 (0 0) 00000080:00200000:1.0:1775764554.997267:0:53518:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764554.997394:0:53518:0:(namei.c:147:ll_iget()) got inode: [0x200003ab1:0x29:0x0](ffff8b2389909988): rc = 0 00000080:00200000:1.0:1775764554.997420:0:53518:0:(file.c:6766:ll_layout_conf()) [0x200003ab1:0x29:0x0]: layout version change: 2 -> 2 00000080:00010000:1.0:1775764554.997431:0:53518:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x29:0x0](ffff8b2389909988) for lock 0x73cfd86cfc17b3b3 00000080:00010000:1.0:1775764554.997450:0:53518:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x29:0x0](ffff8b2389909988) 00000080:00010000:1.0:1775764554.997514:0:53518:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3b3 from it ffffa60b01e5bb20 00010000:00010000:1.0:1775764554.997518:0:53518:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764554.997529:0:53518:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:1.0:1775764554.997593:0:53518:0:(file.c:6226:ll_getattr_dentry()) START file f21a.sanity-pcc:[0x200003ab1:0x29:0x0](ffff8b2389909988), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764554.997601:0:53518:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab1:0x29:0x0](ffff8b2389909988),name=f21a.sanity-pcc 00000002:00010000:1.0:1775764554.997640:0:53518:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x29:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764554.997652:0:53518:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764554.997665:0:53518:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764554.997673:0:53518:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764554.997679:0:53518:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x29:0x0](ffff8b2389909988) 00000080:00010000:1.0:1775764554.997683:0:53518:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x29:0x0](ffff8b2389909988) for lock 0x73cfd86cfc17b3b3 00000080:00010000:1.0:1775764554.997687:0:53518:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3b3 from it ffffa60b01e5bcb8 00010000:00010000:1.0:1775764554.997690:0:53518:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764554.997698:0:53518:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764554.997783:0:53518:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764554.997789:0:53518:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764554.997796:0:53518:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764554.997804:0:53518:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764554.997810:0:53518:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764554.997814:0:53518:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764554.997816:0:53518:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b397 00000080:00010000:1.0:1775764554.997820:0:53518:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b397 from it ffffa60b01e5bbf8 00010000:00010000:1.0:1775764554.997823:0:53518:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764554.997830:0:53518:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764554.997837:0:53518:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764554.997857:0:53518:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21a.sanity-pcc, flags=265 00000080:00200000:1.0:1775764554.997863:0:53518:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab1:0x29:0x0](ffff8b2389909988), inode mode 81a4 mask 46 00000080:00200008:1.0:1775764554.997872:0:53518:0:(file.c:1057:ll_file_open()) START file f21a.sanity-pcc:[0x200003ab1:0x29:0x0](ffff8b2389909988), flags 154402 00000002:00010000:1.0:1775764554.997889:0:53518:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x29:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240114403 00010000:00010000:1.0:1775764554.997943:0:53518:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b3ba lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 53518 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764554.997951:0:53518:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b3ba lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 53518 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764554.997965:0:53518:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764554.997972:0:53518:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ff480 x1862022098423296/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/131984 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764554.997982:0:53518:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b3ba lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 53518 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764554.998009:0:53518:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:53518:1862022098423296:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764554.998244:0:53518:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2382bb59c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764555.004694:0:53518:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:53518:1862022098423296:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764555.004725:0:53518:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b3ba lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fc05 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764555.004746:0:53518:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b3ba lrc: 4/1,0 mode: --/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fc05 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764555.004754:0:53518:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b3ba lrc: 4/1,0 mode: --/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fc05 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764555.004773:0:53518:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b3ba lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fc05 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764555.004786:0:53518:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b3ba lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fc05 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764555.004802:0:53518:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b3ba lrc: 5/2,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fc05 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764555.004812:0:53518:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b3ba lrc: 5/2,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fc05 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764555.004820:0:53518:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b3ba lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fc05 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764555.004833:0:53518:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23864ff480 x1862022098423296/t34359738791(34359738791) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764570 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764555.006645:0:53518:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864ff480 x1862022098423296/t34359738791(34359738791) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764570 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764555.006669:0:53518:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b3ba lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fc05 expref: -99 pid: 53518 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764555.006693:0:53518:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b3ba lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fc05 expref: -99 pid: 53518 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764555.006715:0:53518:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764555.006732:0:53518:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764555.006741:0:53518:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b3ba lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fc05 expref: -99 pid: 53518 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764555.006748:0:53518:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b3ba lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e0304fc05 expref: -99 pid: 53518 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764555.006754:0:53518:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b3ba lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e0304fc05 expref: -99 pid: 53518 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764555.007194:0:53518:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:1.0:1775764555.007206:0:53518:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764555.007353:0:53518:0:(file.c:6766:ll_layout_conf()) [0x200003ab1:0x29:0x0]: layout version change: 2 -> 2 00000080:00010000:1.0:1775764555.007365:0:53518:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x29:0x0](ffff8b2389909988) for lock 0x73cfd86cfc17b3b3 00000080:00010000:1.0:1775764555.007376:0:53518:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3b3 from it ffffa60b01e5bbf0 00010000:00010000:1.0:1775764555.007380:0:53518:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764555.007391:0:53518:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b3b3 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x29:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fbf7 expref: -99 pid: 53518 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000002:00100000:1.0:1775764555.007426:0:53518:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864ff480 x1862022098423296/t34359738791(34359738791) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764570 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764555.007576:0:53518:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab1:0x29:0x0](ffff8b2389909988) cmd=c00466fc arg=7ffd2a497a2c 00000080:00200008:1.0:1775764555.007607:0:53518:0:(file.c:447:ll_file_release()) START file f21a.sanity-pcc:[0x200003ab1:0x29:0x0](ffff8b2389909988), flags 154002 00000002:00100000:1.0:1775764555.007656:0:53518:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864ff480 x1862022098423296/t34359738791(34359738791) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764570 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764555.007666:0:53518:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764555.007671:0:53518:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fca80 x1862022098423552/t0(0) o35->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:23/10 lens 392/752 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764555.007693:0:53518:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:53518:1862022098423552:192.168.202.128@tcp:35:lfs.0 00000100:00100000:1.0:1775764555.007793:0:53518:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2382bb5b40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764555.027476:0:53518:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:53518:1862022098423552:192.168.202.128@tcp:35:lfs.0 00000002:00100000:1.0:1775764555.027529:0:53518:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864ff480 x1862022098423296/t34359738791(34359738791) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764570 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764555.626119:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764558.408444:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2376:1862022098425600:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764558.408553:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098425856:192.168.202.128@tcp:400:kworker.0 00010000:00010000:1.0:1775764560.697730:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1775764562.383437:0:54004:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764562.383485:0:54004:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764562.383519:0:54004:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764562.383542:0:54004:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:1.0:1775764562.383561:0:54004:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:1.0:1775764562.383576:0:54004:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764562.383588:0:54004:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764562.383591:0:54004:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b397 00000080:00010000:1.0:1775764562.383600:0:54004:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b397 from it ffffa60b01cf3cb8 00010000:00010000:1.0:1775764562.383604:0:54004:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:1.0:1775764562.383613:0:54004:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b397 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fb5d expref: -99 pid: 53495 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00000080:00200008:1.0:1775764562.385299:0:54004:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764562.385320:0:54004:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:1.0:1775764562.385347:0:54004:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764562.385478:0:54004:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740a00/0x73cfd86cfc17b3d6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 54004 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764562.385491:0:54004:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740a00/0x73cfd86cfc17b3d6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 54004 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764562.385504:0:54004:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740a00/0x73cfd86cfc17b3d6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 54004 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764562.385542:0:54004:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job lsof:82781e18-e0e3-4f9f-9a2b-041bf0344414:54004:1862022098426624:192.168.202.128@tcp:101:lsof.0 00000100:00100000:1.0:1775764562.385714:0:54004:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b6543c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764562.394879:0:54004:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job lsof:82781e18-e0e3-4f9f-9a2b-041bf0344414:54004:1862022098426624:192.168.202.128@tcp:101:lsof.0 00010000:00010000:1.0:1775764562.394978:0:54004:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740a00/0x73cfd86cfc17b3d6 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fcad expref: -99 pid: 54004 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764562.395063:0:54004:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740a00/0x73cfd86cfc17b3d6 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fcad expref: -99 pid: 54004 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764562.395178:0:54004:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740a00/0x73cfd86cfc17b3d6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fcad expref: -99 pid: 54004 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764562.395192:0:54004:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740a00/0x73cfd86cfc17b3d6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fcad expref: -99 pid: 54004 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764562.395207:0:54004:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740a00/0x73cfd86cfc17b3d6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fcad expref: -99 pid: 54004 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764562.395215:0:54004:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740a00/0x73cfd86cfc17b3d6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fcad expref: -99 pid: 54004 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764562.395222:0:54004:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740a00/0x73cfd86cfc17b3d6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fcad expref: -99 pid: 54004 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764562.395235:0:54004:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864ff480 x1862022098426624/t0(0) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764578 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lsof.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764562.395250:0:54004:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740a00/0x73cfd86cfc17b3d6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fcad expref: -99 pid: 54004 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764562.395265:0:54004:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2387c07800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1775764562.395274:0:54004:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764562.395315:0:54004:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:1.0:1775764562.395323:0:54004:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17b3d6 00000080:00010000:1.0:1775764562.395330:0:54004:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:1.0:1775764562.395334:0:54004:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3d6 from it ffffa60b01cf3cb8 00010000:00010000:1.0:1775764562.395337:0:54004:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740a00/0x73cfd86cfc17b3d6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fcad expref: -99 pid: 54004 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764562.395345:0:54004:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740a00/0x73cfd86cfc17b3d6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fcad expref: -99 pid: 54004 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764562.744265:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764565.816668:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764566.850441:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764568.486487:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118999296 00000100:00100000:1.0:1775764568.486499:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118999296 00000100:00100000:1.0:1775764568.486550:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 137 00000100:00100000:1.0:1775764568.486559:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daf480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42324:x1862022118999296:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764568.486580:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740a00/0x73cfd86cfc17b3d6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e0304fcad expref: -99 pid: 54004 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764568.486662:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764568.486668:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daf480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42324:x1862022118999296:12345-192.168.202.128@tcp:104: Request processed in 111us (319us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764568.486675:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 137 00000100:00100000:1.0:1775764568.513061:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022118999424 00000100:00100000:1.0:1775764568.513073:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022118999424 00000100:00100000:1.0:1775764568.513133:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 138 00000100:00100000:1.0:1775764568.513144:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dae300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:44493:x1862022118999424:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764568.513165:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030000/0x73cfd86cfc17b3e4 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0304fcde expref: -99 pid: 54259 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764568.513299:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764568.513306:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dae300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:44493:x1862022118999424:12345-192.168.202.128@tcp:104: Request processed in 165us (455us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764568.513314:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 138 00000100:00100000:1.0:1775764568.705331:0:54267:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:54267:1862022098429696:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764568.705364:0:54267:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51c00/0x73cfd86cfc17b400 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd4e expref: -99 pid: 54267 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.705408:0:54267:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51c00/0x73cfd86cfc17b400 lrc: 4/1,0 mode: --/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd4e expref: -99 pid: 54267 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.705418:0:54267:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51c00/0x73cfd86cfc17b400 lrc: 4/1,0 mode: --/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd4e expref: -99 pid: 54267 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.705428:0:54267:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51c00/0x73cfd86cfc17b400 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd4e expref: -99 pid: 54267 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.705439:0:54267:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51c00/0x73cfd86cfc17b400 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd4e expref: -99 pid: 54267 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.705450:0:54267:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51c00/0x73cfd86cfc17b400 lrc: 5/2,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd4e expref: -99 pid: 54267 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.705497:0:54267:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51c00/0x73cfd86cfc17b400 lrc: 5/2,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd4e expref: -99 pid: 54267 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.705511:0:54267:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51c00/0x73cfd86cfc17b400 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd4e expref: -99 pid: 54267 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764568.705523:0:54267:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b2382dac380 x1862022098429696/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 584/872 e 0 to 0 dl 1775764584 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764568.705535:0:54267:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 288 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51c00/0x73cfd86cfc17b400 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd4e expref: -99 pid: 54267 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764568.705549:0:54267:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51c00/0x73cfd86cfc17b400 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd4e expref: -99 pid: 54267 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.705560:0:54267:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 4 res 8589949617/42 (0 0) 00000080:00200000:1.0:1775764568.705569:0:54267:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764568.705662:0:54267:0:(namei.c:147:ll_iget()) got inode: [0x200003ab1:0x2a:0x0](ffff8b2390ec00c8): rc = 0 00000080:00200000:1.0:1775764568.705685:0:54267:0:(file.c:6766:ll_layout_conf()) [0x200003ab1:0x2a:0x0]: layout version change: 4294967294 -> 2 00000080:00010000:1.0:1775764568.705694:0:54267:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x2a:0x0](ffff8b2390ec00c8) for lock 0x73cfd86cfc17b400 00000080:00010000:1.0:1775764568.705708:0:54267:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x2a:0x0](ffff8b2390ec00c8) 00000080:00010000:1.0:1775764568.705712:0:54267:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b400 from it ffffa60b00ddbb20 00010000:00010000:1.0:1775764568.705715:0:54267:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51c00/0x73cfd86cfc17b400 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0304fd4e expref: -99 pid: 54267 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.705725:0:54267:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51c00/0x73cfd86cfc17b400 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fd4e expref: -99 pid: 54267 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:1.0:1775764568.705769:0:54267:0:(file.c:6226:ll_getattr_dentry()) START file f21b.sanity-pcc:[0x200003ab1:0x2a:0x0](ffff8b2390ec00c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764568.705776:0:54267:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab1:0x2a:0x0](ffff8b2390ec00c8),name=f21b.sanity-pcc 00000002:00010000:1.0:1775764568.705799:0:54267:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x2a:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764568.705809:0:54267:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51c00/0x73cfd86cfc17b400 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fd4e expref: -99 pid: 54267 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764568.705821:0:54267:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51c00/0x73cfd86cfc17b400 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fd4e expref: -99 pid: 54267 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764568.705827:0:54267:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764568.705832:0:54267:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x2a:0x0](ffff8b2390ec00c8) 00000080:00010000:1.0:1775764568.705835:0:54267:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x2a:0x0](ffff8b2390ec00c8) for lock 0x73cfd86cfc17b400 00000080:00010000:1.0:1775764568.705839:0:54267:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b400 from it ffffa60b00ddbcb8 00010000:00010000:1.0:1775764568.705842:0:54267:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51c00/0x73cfd86cfc17b400 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0304fd4e expref: -99 pid: 54267 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764568.705848:0:54267:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51c00/0x73cfd86cfc17b400 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fd4e expref: -99 pid: 54267 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764568.705890:0:54267:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x2a:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1775764568.705901:0:54267:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab1:0x2a:0x0]: flr state: 1, move mirror from 1 to 1, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764568.705905:0:54267:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1775764568.705909:0:54267:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab1:0x2a:0x0] 00000008:00010000:1.0:1775764568.705941:0:54267:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764568.705945:0:54267:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2384cd0db0, osc lock ffff8b2382cf77c0, flags 0x400000400001002 00000008:00010000:1.0:1775764568.705964:0:54267:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764568.705966:0:54267:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2384cd0df0, osc lock ffff8b2382cf71d0, flags 0x400000400001002 00010000:00010000:1.0:1775764568.706021:0:54267:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000006bafc880/0x73cfd86cfc17b407 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.706031:0:54267:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000006bafc880/0x73cfd86cfc17b407 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.706057:0:54267:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff8b23864ff480 x1862022098429824/t0(0) o101->lustre-OST0001-osc-ffff8b2384b87800@192.168.202.128@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764568.706119:0:54267:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ae897c9/0x73cfd86cfc17b40e lrc: 3/1,0 mode: --/PR res: [0x240000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.706128:0:54267:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ae897c9/0x73cfd86cfc17b40e lrc: 3/1,0 mode: --/PR res: [0x240000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.706138:0:54267:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff8b23864fdf80 x1862022098429952/t0(0) o101->lustre-OST0000-osc-ffff8b2384b87800@192.168.202.128@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000020:00200000:1.0:1775764568.706152:0:54267:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00010000:00010000:1.0:1775764568.715663:0:2377:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ae897c9/0x73cfd86cfc17b40e lrc: 6/2,0 mode: --/PR res: [0x240000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.715680:0:2377:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ae897c9/0x73cfd86cfc17b40e lrc: 6/2,0 mode: --/PR res: [0x240000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd5c expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.715690:0:2377:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ae897c9/0x73cfd86cfc17b40e lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd5c expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.715699:0:2377:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ae897c9/0x73cfd86cfc17b40e lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd5c expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1775764568.715710:0:2377:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ae897c9/0x73cfd86cfc17b40e lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0304fd5c expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1775764568.715718:0:2377:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1775764568.715778:0:2377:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ae897c9/0x73cfd86cfc17b40e lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304fd5c expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.715787:0:2377:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ae897c9/0x73cfd86cfc17b40e lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304fd5c expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.715793:0:2377:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ae897c9/0x73cfd86cfc17b40e lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304fd5c expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.715800:0:2377:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000006ae897c9/0x73cfd86cfc17b40e lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x10d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304fd5c expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1775764568.715812:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098429952:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764568.749095:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098430720:192.168.202.128@tcp:101:lfs.0 00000080:00200000:1.0:1775764568.759315:0:54267:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x2a:0x0] updating i_size 0 i_blocks 2 00000020:00010000:1.0:1775764568.759322:0:54267:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2398371b60 (ffff8b2383fb5ea8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764568.759325:0:54267:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2398371b60 (ffff8b2383fb5ea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764568.759918:0:54267:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238b6b0870 (ffff8b2383fb5f38/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764568.759924:0:54267:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238b6b08b0 (ffff8b2383fb5bd8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764568.759927:0:54267:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2398371b60 (ffff8b2383fb5ea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764568.760011:0:54267:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238b6b0870 (ffff8b2383fb5ea8/1) at cl_lock_fini():72 00000020:00010000:1.0:1775764568.760017:0:54267:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238b6b08b0 (ffff8b2383fb5ea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764568.760028:0:54267:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:0 00000080:00200000:1.0:1775764568.957545:0:54278:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764568.957571:0:54278:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764568.957590:0:54278:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764568.957605:0:54278:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764568.957614:0:54278:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764568.957621:0:54278:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764568.957624:0:54278:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b3f2 00000080:00010000:1.0:1775764568.957631:0:54278:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3f2 from it ffffa60b01e2bbf8 00010000:00010000:1.0:1775764568.957634:0:54278:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764568.957643:0:54278:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764568.957658:0:54278:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764568.957670:0:54278:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f21b.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), file ffff8b23919e6f00, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764568.957676:0:54278:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f21b.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=open 00000002:00010000:1.0:1775764568.957689:0:54278:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f21b.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764568.957735:0:54278:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b431 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 54278 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764568.957742:0:54278:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b431 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 54278 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764568.957758:0:54278:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764568.957770:0:54278:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fdf80 x1862022098431488/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 816/132016 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764568.957779:0:54278:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b431 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 54278 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764568.957799:0:54278:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:54278:1862022098431488:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764568.957960:0:54278:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384cd0cc0 going to sleep for 11 seconds 00010000:00010000:1.0:1775764569.078408:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764569.078418:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030000/0x73cfd86cfc17b43f lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304fdfd expref: -99 pid: 54284 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764569.078427:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2380030000 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764569.078432:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030000/0x73cfd86cfc17b43f lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304fdfd expref: -99 pid: 54284 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764569.078441:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030000/0x73cfd86cfc17b43f lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304fdfd expref: -99 pid: 54284 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764569.078449:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030000/0x73cfd86cfc17b43f lrc: 3/0,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0304fdfd expref: -99 pid: 54284 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764569.078510:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50e00/0x73cfd86cfc17b438 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fdef expref: -99 pid: 54284 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764569.078552:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030000/0x73cfd86cfc17b43f lrc: 2/0,0 mode: --/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304fdfd expref: -99 pid: 54284 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764569.078563:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764569.078587:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030000/0x73cfd86cfc17b43f lrc: 1/0,0 mode: --/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304fdfd expref: -99 pid: 54284 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764569.078594:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030000/0x73cfd86cfc17b43f lrc: 0/0,0 mode: --/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304fdfd expref: -99 pid: 54284 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764569.078608:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764569.078610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764569.078611:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764569.102971:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098432384:192.168.202.128@tcp:103:ldlm_bl.0 00000080:00200008:1.0:1775764569.409745:0:54301:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764569.409759:0:54301:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764569.409782:0:54301:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764569.409805:0:54301:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764569.409820:0:54301:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1775764569.409827:0:54301:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764569.409835:0:54301:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764569.409838:0:54301:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b3f2 00000080:00010000:1.0:1775764569.409845:0:54301:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3f2 from it ffffa60b01defcb8 00010000:00010000:1.0:1775764569.409849:0:54301:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764569.409863:0:54301:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:1.0:1775764569.409889:0:54301:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764569.409895:0:54301:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764569.409902:0:54301:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764569.409910:0:54301:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:1.0:1775764569.409917:0:54301:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764569.409921:0:54301:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764569.409923:0:54301:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b3f2 00000080:00010000:1.0:1775764569.409926:0:54301:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3f2 from it ffffa60b01defb58 00010000:00010000:1.0:1775764569.409928:0:54301:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764569.409935:0:54301:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1775764569.409943:0:54301:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764569.409952:0:54301:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21b.sanity-pcc, flags=8 00000080:00200008:1.0:1775764569.409958:0:54301:0:(file.c:6226:ll_getattr_dentry()) START file f21b.sanity-pcc:[0x200003ab1:0x2a:0x0](ffff8b2390ec00c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764569.409962:0:54301:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab1:0x2a:0x0](ffff8b2390ec00c8),name=f21b.sanity-pcc 00000002:00010000:1.0:1775764569.409975:0:54301:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x2a:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764569.409983:0:54301:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50e00/0x73cfd86cfc17b438 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fdef expref: -99 pid: 54284 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764569.409991:0:54301:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50e00/0x73cfd86cfc17b438 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fdef expref: -99 pid: 54284 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764569.409996:0:54301:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764569.410000:0:54301:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x2a:0x0](ffff8b2390ec00c8) 00000080:00010000:1.0:1775764569.410002:0:54301:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x2a:0x0](ffff8b2390ec00c8) for lock 0x73cfd86cfc17b438 00000080:00010000:1.0:1775764569.410006:0:54301:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b438 from it ffffa60b01defcb8 00010000:00010000:1.0:1775764569.410008:0:54301:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50e00/0x73cfd86cfc17b438 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0304fdef expref: -99 pid: 54284 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764569.410015:0:54301:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50e00/0x73cfd86cfc17b438 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fdef expref: -99 pid: 54284 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764569.410081:0:54301:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x2a:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1775764569.410105:0:54301:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab1:0x2a:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 3 00020000:00200000:1.0:1775764569.410109:0:54301:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1775764569.410113:0:54301:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab1:0x2a:0x0] 00000008:00010000:1.0:1775764569.410149:0:54301:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764569.410154:0:54301:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2384cd00f0, osc lock ffff8b2382cf6d10, flags 0x400000400001002 00000008:00010000:1.0:1775764569.410176:0:54301:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764569.410177:0:54301:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2384cd0130, osc lock ffff8b2382cf70a0, flags 0x400000400001002 00010000:00010000:1.0:1775764569.410196:0:54301:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000ef808137/0x73cfd86cfc17b423 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x10c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304fd8d expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764569.410208:0:54301:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000ef808137/0x73cfd86cfc17b423 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x10c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304fd8d expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1775764569.410226:0:54301:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764569.410230:0:54301:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000ef808137/0x73cfd86cfc17b423 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x10c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304fd8d expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764569.410238:0:54301:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000ef808137/0x73cfd86cfc17b423 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x10c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304fd8d expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764569.410254:0:54301:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 0000000013ca9af5/0x73cfd86cfc17b42a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x10c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304fd94 expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764569.410262:0:54301:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 0000000013ca9af5/0x73cfd86cfc17b42a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x10c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304fd94 expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1775764569.410269:0:54301:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764569.410271:0:54301:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 0000000013ca9af5/0x73cfd86cfc17b42a lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x10c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304fd94 expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764569.410278:0:54301:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 0000000013ca9af5/0x73cfd86cfc17b42a lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x10c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304fd94 expref: -99 pid: 54267 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1775764569.410285:0:54301:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764569.410308:0:54301:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x2a:0x0] updating i_size 0 i_blocks 2 00000020:00010000:1.0:1775764569.410312:0:54301:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f18db0 (ffff8b2387cbaa28/0) at cl_lock_release():245 00000020:00010000:1.0:1775764569.410315:0:54301:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f18db0 (ffff8b2387cbaa28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764569.410318:0:54301:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384cd00f0 (ffff8b2387cba7e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764569.410322:0:54301:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384cd0130 (ffff8b2387cba2d8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764569.410325:0:54301:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f18db0 (ffff8b2387cbaa28/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764569.410328:0:54301:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384cd00f0 (ffff8b2387cbaa28/1) at cl_lock_fini():72 00000020:00010000:1.0:1775764569.410331:0:54301:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384cd0130 (ffff8b2387cbaa28/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764569.410382:0:54301:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x2a:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764569.410415:0:54301:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764569.410423:0:54301:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764569.410438:0:54301:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764569.410447:0:54301:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1775764569.411723:0:54301:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764569.411732:0:54301:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764569.411736:0:54301:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b3f2 00000080:00010000:1.0:1775764569.411744:0:54301:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3f2 from it ffffa60b01defbf8 00010000:00010000:1.0:1775764569.411748:0:54301:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764569.411756:0:54301:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:1.0:1775764569.411771:0:54301:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764569.411778:0:54301:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21b.sanity-pcc, flags=265 00000080:00200000:1.0:1775764569.411785:0:54301:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab1:0x2a:0x0](ffff8b2390ec00c8), inode mode 81a4 mask 44 00000080:00200008:1.0:1775764569.411792:0:54301:0:(file.c:1057:ll_file_open()) START file f21b.sanity-pcc:[0x200003ab1:0x2a:0x0](ffff8b2390ec00c8), flags 100000 00000002:00010000:1.0:1775764569.411811:0:54301:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x2a:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0240100001 00010000:00010000:1.0:1775764569.411862:0:54301:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741000/0x73cfd86cfc17b446 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 54301 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764569.411869:0:54301:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741000/0x73cfd86cfc17b446 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 54301 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764569.411879:0:54301:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764569.411890:0:54301:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ffb80 x1862022098432768/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 928/132136 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764569.411900:0:54301:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741000/0x73cfd86cfc17b446 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 54301 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764569.411921:0:54301:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:54301:1862022098432768:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764569.412316:0:54301:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384cd00c0 going to sleep for 11 seconds 00010000:00010000:1.0:1775764569.437447:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764569.437472:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741000/0x73cfd86cfc17b446 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304fe27 expref: -99 pid: 54301 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764569.437483:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8741000 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764569.437488:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741000/0x73cfd86cfc17b446 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304fe27 expref: -99 pid: 54301 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764569.437498:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741000/0x73cfd86cfc17b446 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0304fe27 expref: -99 pid: 54301 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764569.437508:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741000/0x73cfd86cfc17b446 lrc: 3/0,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0304fe27 expref: -99 pid: 54301 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764569.437539:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50e00/0x73cfd86cfc17b438 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fdef expref: -99 pid: 54284 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764569.437576:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741000/0x73cfd86cfc17b446 lrc: 2/0,0 mode: --/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304fe27 expref: -99 pid: 54301 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764569.437583:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764569.437605:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741000/0x73cfd86cfc17b446 lrc: 1/0,0 mode: --/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304fe27 expref: -99 pid: 54301 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764569.437613:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741000/0x73cfd86cfc17b446 lrc: 0/0,0 mode: --/PR res: [0x200003ab1:0x2a:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304fe27 expref: -99 pid: 54301 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764569.437628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764569.437629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764569.437631:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764569.437640:0:2376:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764569.437652:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098433152:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764569.829841:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:1.0:1775764569.842437:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764569.842489:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23b9ff0450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42326:x1862022118999680:12345-192.168.202.128@tcp:104: Request processed in 10953us (12629us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764569.842505:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 139 00010000:00010000:1.0:1775764571.981163:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764572.985567:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764573.753741:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098434432:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764573.754243:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098434688:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764573.754291:0:2377:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:1.0:1775764573.754299:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098434304:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764573.754330:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098434560:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764573.754360:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098434816:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764573.764728:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098434304:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764573.764762:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098434560:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764573.764773:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098434816:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764573.764787:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098434432:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764573.764799:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098434688:192.168.202.128@tcp:400:kworker.0 00010000:00010000:1.0:1775764574.007634:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764575.031774:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764576.055729:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1775764577.180842:0:54753:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764577.180932:0:54753:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764577.180956:0:54753:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764577.180984:0:54753:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:1.0:1775764577.181002:0:54753:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:1.0:1775764577.181150:0:54753:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764577.181162:0:54753:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764577.181166:0:54753:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b3f2 00000080:00010000:1.0:1775764577.181175:0:54753:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3f2 from it ffffa60b00c87cb8 00010000:00010000:1.0:1775764577.181179:0:54753:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:1.0:1775764577.181188:0:54753:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200008:1.0:1775764577.182208:0:54753:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764577.182215:0:54753:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764577.182229:0:54753:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764577.182242:0:54753:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:1.0:1775764577.182254:0:54753:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:1.0:1775764577.182263:0:54753:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764577.182269:0:54753:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764577.182273:0:54753:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b3f2 00000080:00010000:1.0:1775764577.182279:0:54753:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3f2 from it ffffa60b00c87cb8 00010000:00010000:1.0:1775764577.182282:0:54753:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:1.0:1775764577.182289:0:54753:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:1.0:1775764577.183066:0:54753:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764577.183080:0:54753:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764577.183091:0:54753:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:1.0:1775764577.183103:0:54753:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:1.0:1775764577.183111:0:54753:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764577.183117:0:54753:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764577.183120:0:54753:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b3f2 00000080:00010000:1.0:1775764577.183124:0:54753:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3f2 from it ffffa60b00c87bb0 00010000:00010000:1.0:1775764577.183128:0:54753:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:1.0:1775764577.183135:0:54753:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:1.0:1775764577.183146:0:54753:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764577.183156:0:54753:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 504000 00000080:00200080:1.0:1775764577.183185:0:54753:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=8080667f arg=7fffbd109780 00000080:00200008:1.0:1775764577.183202:0:54753:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 504000 00000080:00200000:1.0:1775764577.183681:0:54753:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764577.183691:0:54753:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764577.183699:0:54753:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:1.0:1775764577.183710:0:54753:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:1.0:1775764577.183716:0:54753:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764577.183720:0:54753:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764577.183722:0:54753:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b3f2 00000080:00010000:1.0:1775764577.183726:0:54753:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3f2 from it ffffa60b00c87bb0 00010000:00010000:1.0:1775764577.183729:0:54753:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:1.0:1775764577.183736:0:54753:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:1.0:1775764577.183747:0:54753:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764577.183752:0:54753:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 504000 00000080:00200080:1.0:1775764577.183760:0:54753:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=8080667f arg=7fffbd108660 00000080:00200008:1.0:1775764577.183768:0:54753:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 504000 00000080:00200000:1.0:1775764577.183894:0:54753:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764577.183902:0:54753:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764577.183910:0:54753:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1775764577.183919:0:54753:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:1.0:1775764577.183926:0:54753:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764577.183930:0:54753:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764577.183932:0:54753:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b3f2 00000080:00010000:1.0:1775764577.183936:0:54753:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3f2 from it ffffa60b00c87bb0 00010000:00010000:1.0:1775764577.183938:0:54753:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1775764577.183949:0:54753:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:1.0:1775764577.183958:0:54753:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764577.183962:0:54753:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 100000 00000080:00200080:1.0:1775764577.184192:0:54753:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=c01466fc arg=7fffbd106540 00000080:00200008:1.0:1775764577.184297:0:54753:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 100000 00000080:00200008:1.0:1775764577.718162:0:54760:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764577.718179:0:54760:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764577.718205:0:54760:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764577.718231:0:54760:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:1.0:1775764577.718249:0:54760:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:1.0:1775764577.718258:0:54760:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764577.718266:0:54760:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764577.718270:0:54760:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b3f2 00000080:00010000:1.0:1775764577.718278:0:54760:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3f2 from it ffffa60b01573cb8 00010000:00010000:1.0:1775764577.718283:0:54760:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:1.0:1775764577.718293:0:54760:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00000080:00200008:1.0:1775764577.718993:0:54760:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764577.719000:0:54760:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:1.0:1775764577.719083:0:54760:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764577.719148:0:54760:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741200/0x73cfd86cfc17b45b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 54760 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764577.719157:0:54760:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741200/0x73cfd86cfc17b45b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 54760 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764577.719165:0:54760:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741200/0x73cfd86cfc17b45b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 54760 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764577.719192:0:54760:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job lsof:82781e18-e0e3-4f9f-9a2b-041bf0344414:54760:1862022098434944:192.168.202.128@tcp:101:lsof.0 00000100:00100000:1.0:1775764577.721428:0:54760:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23ac48a780 going to sleep for 11 seconds 00010000:00010000:1.0:1775764578.103758:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1775764581.592746:0:54919:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-pcc test 21c: Verify HSM release works storing PCC-RO as HSM mirror component ========================================================== 15:56:19 (1775764579) 00000080:00200008:1.0:1775764583.350238:0:55019:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764583.350256:0:55019:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764583.350284:0:55019:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764583.350307:0:55019:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00010000:00010000:1.0:1775764583.350325:0:55019:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00000080:00200000:1.0:1775764583.350335:0:55019:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764583.350344:0:55019:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764583.350351:0:55019:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b3f2 00000080:00010000:1.0:1775764583.350361:0:55019:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3f2 from it ffffa60b01e0bcb8 00010000:00010000:1.0:1775764583.350364:0:55019:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00010000:00010000:1.0:1775764583.350373:0:55019:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00000080:00200008:1.0:1775764583.351559:0:55019:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764583.351566:0:55019:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764583.351578:0:55019:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764583.351590:0:55019:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:1.0:1775764583.351602:0:55019:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00000080:00200000:1.0:1775764583.351609:0:55019:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764583.351614:0:55019:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764583.351617:0:55019:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b3f2 00000080:00010000:1.0:1775764583.351622:0:55019:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3f2 from it ffffa60b01e0bcb8 00010000:00010000:1.0:1775764583.351626:0:55019:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:1.0:1775764583.351633:0:55019:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00000080:00200000:1.0:1775764583.351738:0:55019:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764583.351746:0:55019:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764583.351753:0:55019:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00010000:00010000:1.0:1775764583.351762:0:55019:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00000080:00200000:1.0:1775764583.351768:0:55019:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764583.351773:0:55019:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764583.351775:0:55019:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b3f2 00000080:00010000:1.0:1775764583.351778:0:55019:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3f2 from it ffffa60b01e0bbb0 00010000:00010000:1.0:1775764583.351781:0:55019:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00010000:00010000:1.0:1775764583.351788:0:55019:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00000080:00200000:1.0:1775764583.352537:0:55019:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764583.352547:0:55019:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 100000 00000080:00200000:1.0:1775764583.468167:0:55028:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764583.468207:0:55028:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764583.468226:0:55028:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:1.0:1775764583.468243:0:55028:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00000080:00200000:1.0:1775764583.468251:0:55028:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764583.468261:0:55028:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764583.468265:0:55028:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b3f2 00000080:00010000:1.0:1775764583.468273:0:55028:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3f2 from it ffffa60b01e0bbb0 00010000:00010000:1.0:1775764583.468277:0:55028:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:1.0:1775764583.468286:0:55028:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:1.0:1775764583.468303:0:55028:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764583.468311:0:55028:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 504000 00000080:00200080:1.0:1775764583.468340:0:55028:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=8080667f arg=7ffcf09d2c90 00000080:00200008:1.0:1775764583.468355:0:55028:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 504000 00000080:00200000:1.0:1775764583.528822:0:55034:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764583.528853:0:55034:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764583.528874:0:55034:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:1.0:1775764583.528891:0:55034:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:1.0:1775764583.529036:0:55034:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764583.529044:0:55034:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764583.529048:0:55034:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b3f2 00000080:00010000:1.0:1775764583.529055:0:55034:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3f2 from it ffffa60b01e3bbf8 00010000:00010000:1.0:1775764583.529059:0:55034:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:1.0:1775764583.529068:0:55034:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:1.0:1775764583.529084:0:55034:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764583.529099:0:55034:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764583.529104:0:55034:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764583.529111:0:55034:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1775764583.529119:0:55034:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:1.0:1775764583.529125:0:55034:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764583.529128:0:55034:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764583.529130:0:55034:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b3f2 00000080:00010000:1.0:1775764583.529133:0:55034:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b3f2 from it ffffa60b01e3bb38 00010000:00010000:1.0:1775764583.529135:0:55034:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1775764583.529142:0:55034:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000080:00200000:1.0:1775764583.529150:0:55034:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 3 00000080:00200000:1.0:1775764583.529154:0:55034:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f21c.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), file ffff8b23bad12600, open_flags 8241, mode 81b6 opened 0 00000080:00200000:1.0:1775764583.529161:0:55034:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f21c.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=open|creat 00000002:00010000:1.0:1775764583.529177:0:55034:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f21c.sanity-pcc,[0x200003ab1:0x2b:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0101102 00010000:00010000:1.0:1775764583.529188:0:55034:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:1.0:1775764583.529243:0:55034:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:1.0:1775764583.529249:0:55034:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764583.529252:0:55034:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030a00/0x73cfd86cfc17b3f2 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0304fd08 expref: -99 pid: 54260 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:1.0:1775764583.529285:0:55034:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 55034 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.529292:0:55034:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 55034 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764583.529303:0:55034:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764583.529315:0:55034:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fca80 x1862022098436352/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 936/66600 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764583.529324:0:55034:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 55034 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764583.529346:0:55034:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:55034:1862022098436352:192.168.202.128@tcp:101:bash.0 00000100:00100000:1.0:1775764583.529528:0:55034:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b239118b0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764583.536348:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119001728 00000100:00100000:1.0:1775764583.536360:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119001728 00000100:00100000:1.0:1775764583.536671:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 140 00000100:00100000:1.0:1775764583.536681:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dafb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42325:x1862022119001728:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764583.536698:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741200/0x73cfd86cfc17b45b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e0304fea5 expref: -99 pid: 54760 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764583.536821:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764583.536828:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dafb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42325:x1862022119001728:12345-192.168.202.128@tcp:104: Request processed in 149us (614us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764583.536836:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 140 00000100:00100000:1.0:1775764583.544964:0:55034:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:55034:1862022098436352:192.168.202.128@tcp:101:bash.0 00010000:00010000:1.0:1775764583.544994:0:55034:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.545070:0:55034:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 4/0,1 mode: --/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.545079:0:55034:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 4/0,1 mode: --/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.545091:0:55034:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 4/0,1 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.545104:0:55034:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 4/0,1 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.545116:0:55034:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 5/1,1 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.545124:0:55034:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 5/1,1 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.545136:0:55034:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764583.545148:0:55034:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8b23864fca80 x1862022098436352/t34359738810(34359738810) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 936/664 e 0 to 0 dl 1775764599 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764583.545160:0:55034:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fca80 x1862022098436352/t34359738810(34359738810) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 936/664 e 0 to 0 dl 1775764599 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764583.545175:0:55034:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764583.545189:0:55034:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.545200:0:55034:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 4 res 8589949617/43 (0 0) 00000080:00200000:1.0:1775764583.545207:0:55034:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764583.545224:0:55034:0:(namei.c:1620:ll_create_it()) VFS Op:name=f21c.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=open|creat 00000080:00200000:1.0:1775764583.545267:0:55034:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab1:0x2b:0x0](ffff8b2389a62a08) 00000080:00200000:1.0:1775764583.545473:0:55034:0:(file.c:6766:ll_layout_conf()) [0x200003ab1:0x2b:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1775764583.545486:0:55034:0:(namei.c:1586:ll_create_node()) setting l_ast_data to inode [0x200003ab1:0x2b:0x0](ffff8b2389a62a08) 00000080:00010000:1.0:1775764583.545489:0:55034:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x2b:0x0](ffff8b2389a62a08) for lock 0x73cfd86cfc17b469 00000080:00010000:1.0:1775764583.545501:0:55034:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x2b:0x0](ffff8b2389a62a08) 00000080:00200008:1.0:1775764583.545515:0:55034:0:(file.c:1057:ll_file_open()) START file f21c.sanity-pcc:[0x200003ab1:0x2b:0x0](ffff8b2389a62a08), flags 101101 00000002:00100000:1.0:1775764583.545534:0:55034:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fca80 x1862022098436352/t34359738810(34359738810) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764599 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1775764583.545577:0:55034:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b469 from it ffff8b2389ed6240 00010000:00010000:1.0:1775764583.545582:0:55034:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.545591:0:55034:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764583.545633:0:55034:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab1:0x2b:0x0](ffff8b2389a62a08), inode mode 81a4 mask 40 00000080:00200008:1.0:1775764583.545692:0:55034:0:(file.c:6226:ll_getattr_dentry()) START file f21c.sanity-pcc:[0x200003ab1:0x2b:0x0](ffff8b2389a62a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764583.545698:0:55034:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab1:0x2b:0x0](ffff8b2389a62a08),name=f21c.sanity-pcc 00000002:00010000:1.0:1775764583.545717:0:55034:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x2b:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764583.545726:0:55034:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764583.545736:0:55034:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764583.545742:0:55034:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764583.545746:0:55034:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x2b:0x0](ffff8b2389a62a08) 00000080:00010000:1.0:1775764583.545749:0:55034:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x2b:0x0](ffff8b2389a62a08) for lock 0x73cfd86cfc17b469 00000080:00010000:1.0:1775764583.545753:0:55034:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b469 from it ffffa60b01e3bcd8 00010000:00010000:1.0:1775764583.545755:0:55034:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764583.545762:0:55034:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764583.545799:0:55034:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x2b:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764583.545809:0:55034:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab1:0x2b:0x0] 00000008:00010000:1.0:1775764583.545837:0:55034:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764583.545841:0:55034:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318130, osc lock ffff8b2382cf70a0, flags 0x400001002 00010000:00010000:1.0:1775764583.545890:0:55034:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000002134eda7/0x73cfd86cfc17b470 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 55034 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.545901:0:55034:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000002134eda7/0x73cfd86cfc17b470 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 55034 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1775764583.545956:0:55034:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:1.0:1775764583.545976:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098436608:192.168.202.128@tcp:101:bash.0 00000080:00200000:1.0:1775764583.554103:0:55034:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x2b:0x0] updating i_size 0 i_blocks 1 00000020:00010000:1.0:1775764583.554111:0:55034:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f18db0 (ffff8b2387cbaa28/0) at cl_lock_release():245 00000020:00010000:1.0:1775764583.554115:0:55034:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f18db0 (ffff8b2387cbaa28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764583.554120:0:55034:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318130 (ffff8b2387cba2d8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764583.554125:0:55034:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f18db0 (ffff8b2387cbaa28/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764583.554127:0:55034:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318130 (ffff8b2387cbaa28/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764583.554167:0:55034:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x2b:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1775764583.555071:0:55034:0:(file.c:2612:do_file_write_iter()) START file f21c.sanity-pcc:[0x200003ab1:0x2b:0x0], ppos: 0, count: 17 00000080:00200000:1.0:1775764583.555094:0:55034:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 17 bytes 00000080:00200000:1.0:1775764583.555103:0:55034:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 17 00000080:00200000:1.0:1775764583.555138:0:55034:0:(file.c:2058:ll_file_io_generic()) f21c.sanity-pcc: write ppos: 0, bytes: 17 00000080:00200000:1.0:1775764583.555144:0:55034:0:(file.c:1815:ll_io_set_mirror()) f21c.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764583.555152:0:55034:0:(cl_io.c:214:cl_io_rw_init()) header@000000009288ce5e[0x110, 1, [0x200003ab1:0x2b:0x0] hash] 00000020:00200000:1.0:1775764583.555154:0:55034:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 17) 0 0 00000080:00200000:1.0:1775764583.555159:0:55034:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x2b:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764583.555177:0:55034:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1775764583.555187:0:55034:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 17) 17, 17 00020000:00200000:1.0:1775764583.555192:0:55034:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764583.555220:0:55034:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1775764583.555234:0:55034:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764583.555240:0:55034:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318130, osc lock ffff8b2382cf70a0, flags 0x0 00010000:00010000:1.0:1775764583.555359:0:55034:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b23831ed400 type 11 mode 2 res 9663677440/270 (0 4095) 00010000:00010000:1.0:1775764583.555716:0:55034:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000c62bcd11/0x73cfd86cfc17b477 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x10e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 55034 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.555727:0:55034:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000c62bcd11/0x73cfd86cfc17b477 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x10e:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 55034 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.555738:0:55034:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000c62bcd11/0x73cfd86cfc17b477 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x10e:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 55034 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764583.555763:0:55034:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:55034:1862022098436736:192.168.202.128@tcp:101:bash.0 00000100:00100000:1.0:1775764583.556069:0:55034:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b239118bb40 going to sleep for 11 seconds 00010000:00010000:1.0:1775764583.560993:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764583.561003:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dadc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42945:x1862022119001856:12345-192.168.202.128@tcp:104: Request processed in 782us (1363us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764583.561012:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 141 00000100:00100000:1.0:1775764583.567750:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119001984 00000100:00100000:1.0:1775764583.567765:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119001984 00000100:00100000:1.0:1775764583.567838:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 142 00000100:00100000:1.0:1775764583.567849:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42311:x1862022119001984:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764583.567867:0:46139:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000c62bcd11/0x73cfd86cfc17b477 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fef9 expref: -99 pid: 55034 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.567881:0:46139:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000c62bcd11/0x73cfd86cfc17b477 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304fef9 expref: -99 pid: 55034 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.567900:0:46139:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000c62bcd11/0x73cfd86cfc17b477 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304fef9 expref: -99 pid: 55034 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.567918:0:46139:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000c62bcd11/0x73cfd86cfc17b477 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e0304fef9 expref: -99 pid: 55034 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.567928:0:46139:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000c62bcd11/0x73cfd86cfc17b477 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e0304fef9 expref: -99 pid: 55034 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.567938:0:46139:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000c62bcd11/0x73cfd86cfc17b477 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000000000 nid: local remote: 0x57b8ff0e0304fef9 expref: -99 pid: 55034 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764583.567957:0:46139:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b23b8741800) 00000100:00100000:1.0:1775764583.568021:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42311:x1862022119001984:12345-192.168.202.128@tcp:105: Request processed in 174us (2758us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764583.568029:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 142 00000100:00100000:1.0:1775764584.597053:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764584.597106:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119002880 00000100:00100000:1.0:1775764584.597120:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119002880 00000100:00100000:1.0:1775764584.597206:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 143 00000100:00100000:1.0:1775764584.597220:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23864ff800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42356:x1862022119002880:12345-192.168.202.128@tcp:107:hsm_cdtr.0 00000100:00100000:1.0:1775764584.597417:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23864ff800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42356:x1862022119002880:12345-192.168.202.128@tcp:107:hsm_cdtr.0 Request processed in 199us (380us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764584.597427:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 143 00000080:00200080:1.0:1775764584.623519:0:55048:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=c0086696 arg=7f5e18000b60 00800000:00000080:1.0:1775764584.623545:0:55048:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=c0086696 len=4128 karg=ffff8b2384b0c000 uarg=0000000000000000 00000002:00000080:1.0:1775764584.623554:0:55048:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2384b87800: cmd=c0086696 len=4128 karg=ffff8b2384b0c000 uarg=0000000000000000 00000002:00000080:1.0:1775764584.623561:0:55048:0:(mdc_request.c:1738:mdc_ioc_fid2path()) path get [0x200003ab1:0x2b:0x0] from 18446744073709551615 #0 00000100:00100000:1.0:1775764584.623614:0:55048:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:55048:1862022098437888:192.168.202.128@tcp:53:lhsmtool_posix.0 00000100:00100000:1.0:1775764584.623815:0:55048:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b239118b780 going to sleep for 11 seconds 00000100:00100000:1.0:1775764584.647977:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119003008 00000100:00100000:1.0:1775764584.647990:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119003008 00000100:00100000:1.0:1775764584.648062:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 144 00000100:00100000:1.0:1775764584.648073:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daf480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42945:x1862022119003008:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764584.648097:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000c62bcd11/0x73cfd86cfc17b477 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800420000020000 nid: local remote: 0x57b8ff0e0304fef9 expref: -99 pid: 55034 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764584.648439:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764584.648447:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daf480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42945:x1862022119003008:12345-192.168.202.128@tcp:104: Request processed in 376us (1039us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764584.648482:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 144 00000100:00100000:1.0:1775764584.675021:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098438656:192.168.202.128@tcp:103:ldlm_bl.0 00000080:00200000:1.0:1775764584.708063:0:55048:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x2b:0x0] updating i_size 17 i_blocks 1 00000020:00010000:1.0:1775764584.708077:0:55048:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2389ea8db0 (ffff8b238a3ff008/0) at cl_lock_release():245 00000020:00010000:1.0:1775764584.708083:0:55048:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389ea8db0 (ffff8b238a3ff008/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764584.708547:0:55048:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238649bb30 (ffff8b238a3ff5a8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764584.708555:0:55048:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389ea8db0 (ffff8b238a3ff008/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764584.708558:0:55048:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238649bb30 (ffff8b238a3ff008/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764584.714062:0:55048:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x2b:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:1.0:1775764584.714192:0:55048:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=401866d8 arg=7f5e1f1dc8e0 00800000:00000080:1.0:1775764584.714207:0:55048:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=401866d8 len=64 karg=ffffa60b01e6bcf0 uarg=0000000000000000 00000002:00000080:1.0:1775764584.714221:0:55048:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2384b87800: cmd=401866d8 len=64 karg=ffffa60b01e6bcf0 uarg=0000000000000000 00000020:00100000:1.0:1775764584.714271:0:55048:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 57, max 7 00000100:00100000:1.0:1775764584.714287:0:55048:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fed80 x1862022098439296/t0(0) o57->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764584.714322:0:55048:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:55048:1862022098439296:192.168.202.128@tcp:57:lhsmtool_posix.0 00000100:00100000:1.0:1775764584.714426:0:55048:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b239118b000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764584.719055:0:55048:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:55048:1862022098439296:192.168.202.128@tcp:57:lhsmtool_posix.0 00000080:00200008:1.0:1775764584.719355:0:55048:0:(file.c:2454:do_file_read_iter()) START file 0x200003ab1:0x2b:0x0:[0x200003ab1:0x2b:0x0], ppos: 0, count: 17 00000080:00200000:1.0:1775764584.719384:0:55048:0:(rw.c:1928:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1775764584.719441:0:55048:0:(file.c:2058:ll_file_io_generic()) 0x200003ab1:0x2b:0x0: read ppos: 0, bytes: 17 00000080:00200000:1.0:1775764584.719448:0:55048:0:(file.c:1815:ll_io_set_mirror()) 0x200003ab1:0x2b:0x0: desiginated mirror: 0 00000020:00200000:1.0:1775764584.720929:0:55048:0:(cl_io.c:214:cl_io_rw_init()) header@000000009288ce5e[0x110, 1, [0x200003ab1:0x2b:0x0] hash] 00000020:00200000:1.0:1775764584.720933:0:55048:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 17) 2048 0 00000080:00200000:1.0:1775764584.720940:0:55048:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x2b:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000100:00100000:1.0:1775764584.818670:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200008:1.0:1775764586.904928:0:55063:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764586.904948:0:55063:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764586.905702:0:55063:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764586.905726:0:55063:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764586.905745:0:55063:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764586.905755:0:55063:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764586.905768:0:55063:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764586.905773:0:55063:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b47e 00000080:00010000:1.0:1775764586.905783:0:55063:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b47e from it ffffa60b01eabcb8 00010000:00010000:1.0:1775764586.905789:0:55063:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764586.905799:0:55063:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764586.905847:0:55063:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764586.905854:0:55063:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764586.905860:0:55063:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764586.905869:0:55063:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764586.905889:0:55063:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764586.905893:0:55063:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764586.905896:0:55063:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b47e 00000080:00010000:1.0:1775764586.905899:0:55063:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b47e from it ffffa60b01eabb58 00010000:00010000:1.0:1775764586.905902:0:55063:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764586.905909:0:55063:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764586.905917:0:55063:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764586.905924:0:55063:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21c.sanity-pcc, flags=8 00000080:00200008:1.0:1775764586.905933:0:55063:0:(file.c:6226:ll_getattr_dentry()) START file f21c.sanity-pcc:[0x200003ab1:0x2b:0x0](ffff8b2389a62a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764586.905937:0:55063:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab1:0x2b:0x0](ffff8b2389a62a08),name=f21c.sanity-pcc 00000002:00010000:1.0:1775764586.906359:0:55063:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x2b:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764586.906370:0:55063:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764586.906381:0:55063:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764586.906387:0:55063:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764586.906392:0:55063:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x2b:0x0](ffff8b2389a62a08) 00000080:00010000:1.0:1775764586.906395:0:55063:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x2b:0x0](ffff8b2389a62a08) for lock 0x73cfd86cfc17b469 00000080:00010000:1.0:1775764586.906399:0:55063:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b469 from it ffffa60b01eabcb8 00010000:00010000:1.0:1775764586.906402:0:55063:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764586.906408:0:55063:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740e00/0x73cfd86cfc17b469 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304fee4 expref: -99 pid: 55034 timeout: 0 lvb_type: 3 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764586.906540:0:55063:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x2b:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764586.906552:0:55063:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab1:0x2b:0x0] 00000008:00010000:1.0:1775764586.906586:0:55063:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764586.906591:0:55063:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318130, osc lock ffff8b2382cf6d10, flags 0x400001002 00010000:00010000:1.0:1775764586.906614:0:55063:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b48c lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304ff2a expref: -99 pid: 55048 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764586.906628:0:55063:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b48c lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304ff2a expref: -99 pid: 55048 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1775764586.906644:0:55063:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764586.906648:0:55063:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b48c lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304ff2a expref: -99 pid: 55048 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764586.906656:0:55063:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b48c lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304ff2a expref: -99 pid: 55048 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1775764586.906667:0:55063:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764586.906678:0:55063:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x2b:0x0] updating i_size 17 i_blocks 1 00000020:00010000:1.0:1775764586.906681:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1b230 (ffff8b2387cba518/0) at cl_lock_release():245 00000020:00010000:1.0:1775764586.906684:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1b230 (ffff8b2387cba518/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764586.906686:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318130 (ffff8b2387cba008/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764586.906690:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1b230 (ffff8b2387cba518/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764586.906702:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318130 (ffff8b2387cba518/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764586.906721:0:55063:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x2b:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764586.906759:0:55063:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764586.906767:0:55063:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764586.906774:0:55063:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764586.906782:0:55063:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764586.906789:0:55063:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764586.906793:0:55063:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764586.906795:0:55063:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b47e 00000080:00010000:1.0:1775764586.906798:0:55063:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b47e from it ffffa60b01eabbf8 00010000:00010000:1.0:1775764586.906800:0:55063:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764586.906808:0:55063:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764586.906815:0:55063:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764586.906820:0:55063:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21c.sanity-pcc, flags=265 00000080:00200000:1.0:1775764586.906825:0:55063:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab1:0x2b:0x0](ffff8b2389a62a08), inode mode 81a4 mask 44 00000080:00200008:1.0:1775764586.906834:0:55063:0:(file.c:1057:ll_file_open()) START file f21c.sanity-pcc:[0x200003ab1:0x2b:0x0](ffff8b2389a62a08), flags 100000 00000002:00010000:1.0:1775764586.906852:0:55063:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x2b:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 04240100001 00010000:00010000:1.0:1775764586.907366:0:55063:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b4a1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 55063 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764586.907376:0:55063:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b4a1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 55063 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764586.907388:0:55063:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764586.907402:0:55063:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fed80 x1862022098441472/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 928/132136 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764586.907411:0:55063:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b4a1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 55063 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764586.907434:0:55063:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:55063:1862022098441472:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764586.907911:0:55063:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b239118b9c0 going to sleep for 11 seconds 00000020:00010000:1.0:1775764586.948955:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384cd66c8 (ffff8b238a3ff128/0) at cl_lock_release():245 00000020:00010000:1.0:1775764586.948960:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384cd66c8 (ffff8b238a3ff128/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764586.948965:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238649be30 (ffff8b238a3ff998/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764586.948977:0:55063:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b48c lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304ff2a expref: -99 pid: 55048 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764586.948999:0:55063:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b48c lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304ff2a expref: -99 pid: 55048 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00010000:1.0:1775764586.949015:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384cd66c8 (ffff8b238a3ff128/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764586.949018:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238649be30 (ffff8b238a3ff128/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764586.949028:0:55063:0:(vvp_io.c:517:vvp_io_advance()) advancing 17 bytes 00000080:00200000:1.0:1775764586.949053:0:55063:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x2b:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764586.949058:0:55063:0:(file.c:2206:ll_file_io_generic()) f21c.sanity-pcc: 1 io complete with rc: 0, result: 17, restart: 0 00000080:00200000:1.0:1775764586.949064:0:55063:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 17 00000080:00200008:1.0:1775764586.950361:0:55063:0:(file.c:2454:do_file_read_iter()) START file f21c.sanity-pcc:[0x200003ab1:0x2b:0x0], ppos: 17, count: 1048576 00000080:00200000:1.0:1775764586.950403:0:55063:0:(file.c:2058:ll_file_io_generic()) f21c.sanity-pcc: read ppos: 17, bytes: 1048576 00000080:00200000:1.0:1775764586.950409:0:55063:0:(file.c:1815:ll_io_set_mirror()) f21c.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764586.950416:0:55063:0:(cl_io.c:214:cl_io_rw_init()) header@000000009288ce5e[0x110, 2, [0x200003ab1:0x2b:0x0] hash] 00000020:00200000:1.0:1775764586.950418:0:55063:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [17, 1048593) 0 0 00000080:00200000:1.0:1775764586.950423:0:55063:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x2b:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764586.950439:0:55063:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab1:0x2b:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764586.950442:0:55063:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:1.0:1775764586.950450:0:55063:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 17 chunk: [17, 1048593) 1048593, 1048576 00020000:00200000:1.0:1775764586.950471:0:55063:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764586.950498:0:55063:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 256] 00000008:00010000:1.0:1775764586.950514:0:55063:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764586.950520:0:55063:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318130, osc lock ffff8b2382cf6d10, flags 0x400000000000000 00010000:00010000:1.0:1775764586.950544:0:55063:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b48c lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304ff2a expref: -99 pid: 55048 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764586.950562:0:55063:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1052671) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b48c lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304ff2a expref: -99 pid: 55048 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764586.950581:0:55063:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b48c lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304ff2a expref: -99 pid: 55048 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764586.950590:0:55063:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b48c lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304ff2a expref: -99 pid: 55048 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764586.950597:0:55063:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b48c lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304ff2a expref: -99 pid: 55048 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764586.950606:0:55063:0:(vvp_io.c:826:vvp_io_read_start()) f21c.sanity-pcc: read [17, 1048593) 00000001:00010000:1.0:1775764586.950626:0:55063:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab1:0x2b:0x0] 00000008:00010000:1.0:1775764586.950631:0:55063:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764586.950634:0:55063:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318530, osc lock ffff8b2382cf6390, flags 0x400000400001002 00010000:00010000:1.0:1775764586.950642:0:55063:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b48c lrc: 4/2,0 mode: PR/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304ff2a expref: -99 pid: 55048 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764586.950651:0:55063:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b48c lrc: 4/2,0 mode: PR/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304ff2a expref: -99 pid: 55048 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1775764586.950660:0:55063:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764586.950663:0:55063:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b48c lrc: 7/2,0 mode: PR/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304ff2a expref: -99 pid: 55048 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764586.950670:0:55063:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b48c lrc: 6/1,0 mode: PR/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304ff2a expref: -99 pid: 55048 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1775764586.950682:0:55063:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764586.950689:0:55063:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x2b:0x0] updating i_size 17 i_blocks 1 00000020:00010000:1.0:1775764586.950693:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1b230 (ffff8b2387cba518/0) at cl_lock_release():245 00000020:00010000:1.0:1775764586.950695:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1b230 (ffff8b2387cba518/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764586.950698:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318530 (ffff8b2387cba008/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764586.950702:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1b230 (ffff8b2387cba518/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764586.950704:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318530 (ffff8b2387cba518/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764586.950712:0:55063:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:17 00000020:00010000:1.0:1775764586.950720:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb7998 (ffff8b2387cba518/0) at cl_lock_release():245 00000020:00010000:1.0:1775764586.950722:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb7998 (ffff8b2387cba518/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764586.950724:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318130 (ffff8b2387cba008/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764586.950728:0:55063:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b48c lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0304ff2a expref: -99 pid: 55048 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764586.950737:0:55063:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b48c lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0304ff2a expref: -99 pid: 55048 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00010000:1.0:1775764586.950749:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb7998 (ffff8b2387cba518/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764586.950752:0:55063:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318130 (ffff8b2387cba518/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764586.950756:0:55063:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:1.0:1775764586.951300:0:55063:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x2b:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764586.951305:0:55063:0:(file.c:2206:ll_file_io_generic()) f21c.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:1.0:1775764586.951308:0:55063:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:1.0:1775764586.951598:0:55063:0:(file.c:447:ll_file_release()) START file f21c.sanity-pcc:[0x200003ab1:0x2b:0x0](ffff8b2389a62a08), flags 100000 00000002:00100000:1.0:1775764586.951668:0:55063:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fed80 x1862022098441472/t34359738817(34359738817) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764602 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764586.951686:0:55063:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764586.951692:0:55063:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fe300 x1862022098442112/t0(0) o35->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:23/10 lens 392/904 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764586.951719:0:55063:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:55063:1862022098442112:192.168.202.128@tcp:35:lfs.0 00000100:00100000:1.0:1775764586.951907:0:55063:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b239118b780 going to sleep for 11 seconds 00000002:00010000:1.0:1775764587.020892:0:55066:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4af lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ffa8 expref: -99 pid: 55066 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764587.020925:0:55066:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 4 res 8589949617/43 (0 0) 00000080:00200000:1.0:1775764587.020933:0:55066:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764587.021030:0:55066:0:(namei.c:147:ll_iget()) got inode: [0x200003ab1:0x2b:0x0](ffff8b2389a62a08): rc = 0 00000080:00200000:1.0:1775764587.021050:0:55066:0:(file.c:6766:ll_layout_conf()) [0x200003ab1:0x2b:0x0]: layout version change: 2 -> 2 00000080:00010000:1.0:1775764587.021060:0:55066:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x2b:0x0](ffff8b2389a62a08) for lock 0x73cfd86cfc17b4af 00000080:00010000:1.0:1775764587.021071:0:55066:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x2b:0x0](ffff8b2389a62a08) 00000080:00200008:1.0:1775764587.021087:0:55066:0:(file.c:1057:ll_file_open()) START file f21c.sanity-pcc:[0x200003ab1:0x2b:0x0](ffff8b2389a62a08), flags 104000 00000002:00100000:1.0:1775764587.021119:0:55066:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382dafb80 x1862022098442368/t34359738820(34359738820) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764602 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1775764587.021952:0:55066:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b4af from it ffff8b23849c3d20 00010000:00010000:1.0:1775764587.021965:0:55066:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4af lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0304ffa8 expref: -99 pid: 55066 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764587.021981:0:55066:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4af lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304ffa8 expref: -99 pid: 55066 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764587.022030:0:55066:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab1:0x2b:0x0](ffff8b2389a62a08), inode mode 81a4 mask 44 00000080:00200080:1.0:1775764587.022075:0:55066:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab1:0x2b:0x0](ffff8b2389a62a08) cmd=802066d3 arg=7fff6f844e90 00800000:00000080:1.0:1775764587.022097:0:55066:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=802066d3 len=384 karg=ffff8b2383ebda00 uarg=0000000000000000 00000002:00000080:1.0:1775764587.022108:0:55066:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2384b87800: cmd=802066d3 len=384 karg=ffff8b2383ebda00 uarg=0000000000000000 00000100:00100000:1.0:1775764587.022399:0:55066:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:55066:1862022098442496:192.168.202.128@tcp:54:lfs.0 00000100:00100000:1.0:1775764587.022609:0:55066:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b239118b300 going to sleep for 11 seconds 00000080:00200000:1.0:1775764587.178680:0:55080:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764587.178724:0:55080:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764587.178750:0:55080:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764587.178769:0:55080:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00000080:00200000:1.0:1775764587.178778:0:55080:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764587.178788:0:55080:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764587.178791:0:55080:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b47e 00000080:00010000:1.0:1775764587.178905:0:55080:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b47e from it ffffa60b01efbbb0 00010000:00010000:1.0:1775764587.178911:0:55080:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764587.178922:0:55080:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 0 00000080:00200000:1.0:1775764587.178943:0:55080:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764587.178955:0:55080:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 304000 00000080:00200080:1.0:1775764587.178990:0:55080:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=901066fc arg=7ffd18460650 00000080:00200008:1.0:1775764587.179033:0:55080:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 304000 00000080:00200000:1.0:1775764587.270034:0:55087:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764587.270081:0:55087:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764587.270106:0:55087:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 0 00010000:00010000:1.0:1775764587.270127:0:55087:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 0 00000080:00200000:1.0:1775764587.270136:0:55087:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764587.270147:0:55087:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764587.270151:0:55087:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b47e 00000080:00010000:1.0:1775764587.270159:0:55087:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b47e from it ffffa60b01f0bbf8 00010000:00010000:1.0:1775764587.270163:0:55087:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 0 00010000:00010000:1.0:1775764587.270172:0:55087:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1775764587.270192:0:55087:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764587.270200:0:55087:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21c.sanity-pcc, flags=265 00000080:00200000:1.0:1775764587.270205:0:55087:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab1:0x2b:0x0](ffff8b2389a62a08), inode mode 81a4 mask 44 00000080:00200008:1.0:1775764587.270210:0:55087:0:(file.c:1057:ll_file_open()) START file f21c.sanity-pcc:[0x200003ab1:0x2b:0x0](ffff8b2389a62a08), flags 100000 00000002:00010000:1.0:1775764587.270336:0:55087:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x2b:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 04240100001 00010000:00010000:1.0:1775764587.270402:0:55087:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b4b6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 55087 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764587.270411:0:55087:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b4b6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 55087 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764587.270430:0:55087:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764587.270445:0:55087:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fc380 x1862022098443008/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 928/132136 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764587.270491:0:55087:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b4b6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 55087 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764587.270523:0:55087:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job cat:7e6eda44-5508-487e-b17f-ff28ad9d3512:55087:1862022098443008:192.168.202.128@tcp:101:cat.0 00000100:00100000:1.0:1775764587.270768:0:55087:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b239118bb40 going to sleep for 11 seconds 00000080:00200000:1.0:1775764587.299923:0:55088:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764587.299934:0:55088:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764587.299943:0:55088:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764587.299952:0:55088:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:1.0:1775764587.299958:0:55088:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764587.299962:0:55088:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764587.299965:0:55088:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b47e 00000080:00010000:1.0:1775764587.299969:0:55088:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b47e from it ffffa60b01efbb98 00010000:00010000:1.0:1775764587.299971:0:55088:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764587.299978:0:55088:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1775764587.299986:0:55088:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764587.299991:0:55088:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21c.sanity-pcc, flags=8 00000080:00200008:1.0:1775764587.300055:0:55088:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764587.300061:0:55088:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764587.300067:0:55088:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764587.300073:0:55088:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764587.300080:0:55088:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1775764587.300087:0:55088:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764587.300090:0:55088:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764587.300092:0:55088:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b47e 00000080:00010000:1.0:1775764587.300095:0:55088:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b47e from it ffffa60b01efbcb8 00010000:00010000:1.0:1775764587.300098:0:55088:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764587.300105:0:55088:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:1.0:1775764587.300120:0:55088:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764587.300125:0:55088:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764587.300131:0:55088:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764587.300138:0:55088:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:1.0:1775764587.300143:0:55088:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764587.300147:0:55088:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764587.300150:0:55088:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b47e 00000080:00010000:1.0:1775764587.300153:0:55088:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b47e from it ffffa60b01efbb58 00010000:00010000:1.0:1775764587.300156:0:55088:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764587.300162:0:55088:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:1.0:1775764587.300169:0:55088:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764587.300173:0:55088:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21c.sanity-pcc, flags=8 00000080:00200008:1.0:1775764587.300178:0:55088:0:(file.c:6226:ll_getattr_dentry()) START file f21c.sanity-pcc:[0x200003ab1:0x2b:0x0](ffff8b2389a62a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764587.300181:0:55088:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab1:0x2b:0x0](ffff8b2389a62a08),name=f21c.sanity-pcc 00000002:00010000:1.0:1775764587.300190:0:55088:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x2b:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764587.300196:0:55088:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4af lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304ffa8 expref: -99 pid: 55066 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764587.300204:0:55088:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4af lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304ffa8 expref: -99 pid: 55066 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764587.300211:0:55088:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764587.300317:0:55088:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x2b:0x0](ffff8b2389a62a08) 00000080:00010000:1.0:1775764587.300321:0:55088:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x2b:0x0](ffff8b2389a62a08) for lock 0x73cfd86cfc17b4af 00000080:00010000:1.0:1775764587.300325:0:55088:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b4af from it ffffa60b01efbcb8 00010000:00010000:1.0:1775764587.300328:0:55088:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4af lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0304ffa8 expref: -99 pid: 55066 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764587.300337:0:55088:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4af lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304ffa8 expref: -99 pid: 55066 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200008:1.0:1775764587.302050:0:55088:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764587.302059:0:55088:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764587.302079:0:55088:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764587.302094:0:55088:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:1.0:1775764587.302109:0:55088:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:1.0:1775764587.302118:0:55088:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764587.302126:0:55088:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764587.302129:0:55088:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b47e 00000080:00010000:1.0:1775764587.302135:0:55088:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b47e from it ffffa60b01efbcb8 00010000:00010000:1.0:1775764587.302139:0:55088:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:1.0:1775764587.302147:0:55088:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:1.0:1775764587.302213:0:55088:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764587.302324:0:55088:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764587.302332:0:55088:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1775764587.302339:0:55088:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:1.0:1775764587.302349:0:55088:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764587.302353:0:55088:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764587.302356:0:55088:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b47e 00000080:00010000:1.0:1775764587.302359:0:55088:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b47e from it ffffa60b01efbbb0 00010000:00010000:1.0:1775764587.302361:0:55088:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1775764587.302368:0:55088:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:1.0:1775764587.302376:0:55088:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764587.302385:0:55088:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 304000 00000080:00200080:1.0:1775764587.305753:0:55088:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=401866d9 arg=241e2a0 00000002:00010000:1.0:1775764587.305933:0:55088:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x2b:0x0]) in obj [0x200003ab1:0x2b:0x0], intent: open flags 03004040000002 00010000:00010000:1.0:1775764587.305963:0:55088:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b4b6 lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x2b:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e0304ffc4 expref: -99 pid: 55087 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764587.306005:0:55088:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5fc00/0x73cfd86cfc17b4a8 lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x2b:0x0].0x0 bits 0xa/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0304ff8c expref: -99 pid: 55063 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000002:00100000:1.0:1775764587.306077:0:55088:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fc380 x1862022098443008/t34359738822(34359738822) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764603 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cat.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764587.306093:0:55088:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764587.306098:0:55088:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ffb80 x1862022098443264/t0(0) o35->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:23/10 lens 392/904 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764587.306129:0:55088:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:55088:1862022098443264:192.168.202.128@tcp:35:lfs.0 00000100:00100000:1.0:1775764587.306399:0:55088:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b239118b3c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764587.323716:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764587.344639:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764587.344740:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200000:1.0:1775764587.364393:0:55089:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764587.364425:0:55089:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764587.364445:0:55089:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:1.0:1775764587.364502:0:55089:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:1.0:1775764587.364511:0:55089:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764587.364519:0:55089:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764587.364523:0:55089:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b47e 00000080:00010000:1.0:1775764587.364532:0:55089:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b47e from it ffffa60b01f13bf8 00010000:00010000:1.0:1775764587.364536:0:55089:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:1.0:1775764587.364546:0:55089:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:1.0:1775764587.364563:0:55089:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764587.364576:0:55089:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f21c.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), file ffff8b2386290c00, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764587.364581:0:55089:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f21c.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=open 00000002:00010000:1.0:1775764587.364595:0:55089:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f21c.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764587.364651:0:55089:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b4c4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 55089 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764587.364658:0:55089:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b4c4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 55089 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764587.364668:0:55089:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764587.364679:0:55089:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fc700 x1862022098444544/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 936/132136 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764587.364686:0:55089:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b4c4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 55089 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764587.364705:0:55089:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc700 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:55089:1862022098444544:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764587.364899:0:55089:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b239118b600 going to sleep for 11 seconds 00000100:00100000:1.0:1775764587.486673:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764587.626041:0:55106:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0d0cd80 going to sleep for 11 seconds 00000080:00200000:1.0:1775764587.683280:0:55107:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764587.683316:0:55107:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764587.683343:0:55107:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:1.0:1775764587.683363:0:55107:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:1.0:1775764587.683371:0:55107:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764587.683381:0:55107:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764587.683385:0:55107:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b47e 00000080:00010000:1.0:1775764587.683393:0:55107:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b47e from it ffffa60b01f33c10 00010000:00010000:1.0:1775764587.683397:0:55107:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:1.0:1775764587.683414:0:55107:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51e00/0x73cfd86cfc17b47e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304ff07 expref: -99 pid: 55036 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:1.0:1775764587.683432:0:55107:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764587.683446:0:55107:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=f21c.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags=0 00000080:00200000:1.0:1775764587.683480:0:55107:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f21c.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=getattr 00000002:00010000:1.0:1775764587.683494:0:55107:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f21c.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764587.683551:0:55107:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b4d9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 55107 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764587.683560:0:55107:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b4d9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 55107 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764587.683571:0:55107:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b4d9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 55107 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764587.683606:0:55107:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job unlink:7e6eda44-5508-487e-b17f-ff28ad9d3512:55107:1862022098445952:192.168.202.128@tcp:101:unlink.0 00000100:00100000:1.0:1775764587.683710:0:55107:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b239118be40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764587.689836:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200008:1.0:1775764593.954780:0:55547:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764593.954798:0:55547:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764593.954834:0:55547:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764593.954858:0:55547:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4e0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030500b9 expref: -99 pid: 55530 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764593.954874:0:55547:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4e0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030500b9 expref: -99 pid: 55530 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764593.954887:0:55547:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764593.954896:0:55547:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764593.954900:0:55547:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b4e0 00000080:00010000:1.0:1775764593.954908:0:55547:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b4e0 from it ffffa60b01d6bcb8 00010000:00010000:1.0:1775764593.954911:0:55547:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4e0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030500b9 expref: -99 pid: 55530 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764593.954920:0:55547:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4e0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030500b9 expref: -99 pid: 55530 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200008:1.0:1775764593.956316:0:55547:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764593.956329:0:55547:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:1.0:1775764593.956351:0:55547:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764593.956438:0:55547:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741800/0x73cfd86cfc17b4e7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 55547 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764593.956449:0:55547:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741800/0x73cfd86cfc17b4e7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 55547 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764593.956497:0:55547:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741800/0x73cfd86cfc17b4e7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 55547 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764593.956535:0:55547:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job lsof:82781e18-e0e3-4f9f-9a2b-041bf0344414:55547:1862022098447872:192.168.202.128@tcp:101:lsof.0 00000100:00100000:1.0:1775764593.956645:0:55547:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b239118ba80 going to sleep for 11 seconds 00000100:00100000:1.0:1775764593.963336:0:55547:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job lsof:82781e18-e0e3-4f9f-9a2b-041bf0344414:55547:1862022098447872:192.168.202.128@tcp:101:lsof.0 00010000:00010000:1.0:1775764593.963364:0:55547:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741800/0x73cfd86cfc17b4e7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030500c7 expref: -99 pid: 55547 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764593.963377:0:55547:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741800/0x73cfd86cfc17b4e7 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030500c7 expref: -99 pid: 55547 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764593.963391:0:55547:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741800/0x73cfd86cfc17b4e7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030500c7 expref: -99 pid: 55547 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764593.963403:0:55547:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741800/0x73cfd86cfc17b4e7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030500c7 expref: -99 pid: 55547 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764593.963416:0:55547:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741800/0x73cfd86cfc17b4e7 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030500c7 expref: -99 pid: 55547 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764593.963424:0:55547:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741800/0x73cfd86cfc17b4e7 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030500c7 expref: -99 pid: 55547 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764593.963430:0:55547:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741800/0x73cfd86cfc17b4e7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030500c7 expref: -99 pid: 55547 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764593.963440:0:55547:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864fd880 x1862022098447872/t0(0) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764609 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lsof.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764593.963483:0:55547:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741800/0x73cfd86cfc17b4e7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030500c7 expref: -99 pid: 55547 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764593.963500:0:55547:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2387c07800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1775764593.963507:0:55547:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764593.963545:0:55547:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:1.0:1775764593.963553:0:55547:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17b4e7 00000080:00010000:1.0:1775764593.963559:0:55547:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:1.0:1775764593.963563:0:55547:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b4e7 from it ffffa60b01d6bcb8 00010000:00010000:1.0:1775764593.963566:0:55547:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741800/0x73cfd86cfc17b4e7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030500c7 expref: -99 pid: 55547 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764593.963576:0:55547:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741800/0x73cfd86cfc17b4e7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030500c7 expref: -99 pid: 55547 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764596.539291:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764598.583595:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1775764598.858629:0:55707:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-pcc test 21d: Write should invalidate PCC-RO caching ========================================================== 15:56:37 (1775764597) 00000100:00100000:1.0:1775764599.354069:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098448512:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764599.354209:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098448768:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764599.354250:0:2376:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:1.0:1775764599.354257:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2376:1862022098448384:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764599.354286:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098448640:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764599.354308:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2376:1862022098448896:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764599.357881:0:2376:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2384b87800: committing for last_committed 34359738839 gen 1 00000100:00100000:1.0:1775764599.357899:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098448512:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764599.367603:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2376:1862022098448384:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764599.367628:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2376:1862022098448896:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764599.375102:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098448768:192.168.202.128@tcp:400:kworker.0 00000080:00200000:1.0:1775764599.875436:0:36:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x2a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764599.875516:0:36:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab1:0x2a:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 3 00020000:00200000:1.0:1775764599.875520:0:36:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1775764599.875534:0:36:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764599.875609:0:36:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x2a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1775764600.386276:0:55806:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764600.386295:0:55806:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764600.386403:0:55806:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764600.386424:0:55806:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4e0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030500b9 expref: -99 pid: 55530 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764600.386440:0:55806:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4e0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030500b9 expref: -99 pid: 55530 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764600.386447:0:55806:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764600.386840:0:55806:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764600.386846:0:55806:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b4e0 00000080:00010000:1.0:1775764600.386856:0:55806:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b4e0 from it ffffa60b01e23cb8 00010000:00010000:1.0:1775764600.386861:0:55806:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4e0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030500b9 expref: -99 pid: 55530 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764600.386871:0:55806:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4e0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030500b9 expref: -99 pid: 55530 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764600.395271:0:55807:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764600.395301:0:55807:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764600.396913:0:55807:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4e0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030500b9 expref: -99 pid: 55530 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764600.396935:0:55807:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4e0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030500b9 expref: -99 pid: 55530 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764600.396946:0:55807:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764600.396957:0:55807:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764600.396961:0:55807:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b4e0 00000080:00010000:1.0:1775764600.396969:0:55807:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b4e0 from it ffffa60b01e2bbb0 00010000:00010000:1.0:1775764600.396973:0:55807:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4e0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030500b9 expref: -99 pid: 55530 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764600.396982:0:55807:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4e0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030500b9 expref: -99 pid: 55530 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764600.396999:0:55807:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764600.397010:0:55807:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 100000 00000080:00200000:1.0:1775764600.397053:0:55807:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764600.397065:0:55807:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764600.397072:0:55807:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4e0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030500b9 expref: -99 pid: 55530 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764600.397081:0:55807:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4e0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030500b9 expref: -99 pid: 55530 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764600.397087:0:55807:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764600.397091:0:55807:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764600.397103:0:55807:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b4e0 00000080:00010000:1.0:1775764600.397107:0:55807:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b4e0 from it ffffa60b01e2bbf8 00010000:00010000:1.0:1775764600.397110:0:55807:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4e0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030500b9 expref: -99 pid: 55530 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764600.397116:0:55807:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b4e0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030500b9 expref: -99 pid: 55530 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764600.397124:0:55807:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764600.397132:0:55807:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=.lustre, flags=17 00000080:00200000:1.0:1775764600.397138:0:55807:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000002:0x1:0x0](ffff8b23899080c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764600.397150:0:55807:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=fid, dir=[0x200000002:0x1:0x0](ffff8b23899080c8), file ffff8b2383929080, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1775764600.397156:0:55807:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=fid, dir=[0x200000002:0x1:0x0](ffff8b23899080c8), intent=open 00000002:00010000:1.0:1775764600.397170:0:55807:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: fid,[0x0:0x0:0x0]) in obj [0x200000002:0x1:0x0], intent: open flags 0100001 00010000:00010000:1.0:1775764600.397223:0:55807:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b4ee lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 55807 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764600.397231:0:55807:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b4ee lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 55807 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764600.397243:0:55807:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764600.397258:0:55807:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ff100 x1862022098449152/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 928/132136 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764600.397267:0:55807:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b4ee lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 55807 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764600.397295:0:55807:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:55807:1862022098449152:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:1.0:1775764600.431792:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764600.431800:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b4ee lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e030500e3 expref: -99 pid: 55807 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764600.431807:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8740200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764600.431812:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b4ee lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e030500e3 expref: -99 pid: 55807 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764600.431819:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b4ee lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e030500e3 expref: -99 pid: 55807 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764600.431827:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b4ee lrc: 3/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e030500e3 expref: -99 pid: 55807 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764600.431844:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8606000/0x73cfd86cfc17b0cd lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f273 expref: -99 pid: 51094 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764600.431880:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b4ee lrc: 2/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030500e3 expref: -99 pid: 55807 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764600.431887:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764600.431915:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b4ee lrc: 1/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030500e3 expref: -99 pid: 55807 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764600.431923:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b4ee lrc: 0/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030500e3 expref: -99 pid: 55807 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764600.431934:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764600.431937:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764600.431938:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764600.431955:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098449280:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764600.443802:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098449280:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764600.529047:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119007488 00000100:00100000:1.0:1775764600.529058:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119007488 00000100:00100000:1.0:1775764600.529127:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 151 00000100:00100000:1.0:1775764600.529136:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daf480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42326:x1862022119007488:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764600.529154:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741800/0x73cfd86cfc17b4e7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e030500c7 expref: -99 pid: 55547 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764600.529235:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764600.529241:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daf480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42326:x1862022119007488:12345-192.168.202.128@tcp:104: Request processed in 107us (283us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764600.529248:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 151 00000080:00200008:1.0:1775764600.647361:0:55822:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764600.647377:0:55822:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764600.647400:0:55822:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764600.647419:0:55822:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5ea00/0x73cfd86cfc17b503 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050122 expref: -99 pid: 55816 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764600.647434:0:55822:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5ea00/0x73cfd86cfc17b503 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050122 expref: -99 pid: 55816 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764600.647442:0:55822:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764600.647559:0:55822:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764600.647564:0:55822:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b503 00000080:00010000:1.0:1775764600.647570:0:55822:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b503 from it ffffa60b01e33cb8 00010000:00010000:1.0:1775764600.647574:0:55822:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5ea00/0x73cfd86cfc17b503 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050122 expref: -99 pid: 55816 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764600.647583:0:55822:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5ea00/0x73cfd86cfc17b503 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050122 expref: -99 pid: 55816 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764600.647609:0:55822:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764600.647614:0:55822:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764600.647620:0:55822:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5ea00/0x73cfd86cfc17b503 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050122 expref: -99 pid: 55816 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764600.647628:0:55822:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5ea00/0x73cfd86cfc17b503 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050122 expref: -99 pid: 55816 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764600.647700:0:55822:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764600.647705:0:55822:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764600.647708:0:55822:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b503 00000080:00010000:1.0:1775764600.647711:0:55822:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b503 from it ffffa60b01e33b58 00010000:00010000:1.0:1775764600.647714:0:55822:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5ea00/0x73cfd86cfc17b503 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050122 expref: -99 pid: 55816 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764600.647722:0:55822:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5ea00/0x73cfd86cfc17b503 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050122 expref: -99 pid: 55816 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764600.647730:0:55822:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764600.647737:0:55822:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21d.sanity-pcc, flags=8 00000080:00200008:1.0:1775764600.647742:0:55822:0:(file.c:6226:ll_getattr_dentry()) START file f21d.sanity-pcc:[0x200003ab1:0x2d:0x0](ffff8b23899c3248), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764600.647746:0:55822:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab1:0x2d:0x0](ffff8b23899c3248),name=f21d.sanity-pcc 00000002:00010000:1.0:1775764600.647766:0:55822:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x2d:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764600.647774:0:55822:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e600/0x73cfd86cfc17b4f5 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x2d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050106 expref: -99 pid: 55577 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764600.647783:0:55822:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e600/0x73cfd86cfc17b4f5 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x2d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050106 expref: -99 pid: 55577 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764600.647789:0:55822:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764600.647792:0:55822:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x2d:0x0](ffff8b23899c3248) 00000080:00010000:1.0:1775764600.647795:0:55822:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x2d:0x0](ffff8b23899c3248) for lock 0x73cfd86cfc17b4f5 00000080:00010000:1.0:1775764600.647798:0:55822:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b4f5 from it ffffa60b01e33cb8 00010000:00010000:1.0:1775764600.647801:0:55822:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e600/0x73cfd86cfc17b4f5 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x2d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03050106 expref: -99 pid: 55577 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764600.647808:0:55822:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e600/0x73cfd86cfc17b4f5 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x2d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050106 expref: -99 pid: 55577 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764600.647874:0:55822:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x2d:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764600.647883:0:55822:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab1:0x2d:0x0] 00000008:00010000:1.0:1775764600.647908:0:55822:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764600.647911:0:55822:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318030, osc lock ffff8b2382cf6d10, flags 0x400001002 00010000:00010000:1.0:1775764600.647924:0:55822:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 0000000063dd8033/0x73cfd86cfc17b4fc lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x10e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03050114 expref: -99 pid: 55577 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764600.647935:0:55822:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 0000000063dd8033/0x73cfd86cfc17b4fc lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03050114 expref: -99 pid: 55577 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1775764600.647947:0:55822:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764600.647950:0:55822:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 0000000063dd8033/0x73cfd86cfc17b4fc lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03050114 expref: -99 pid: 55577 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764600.647958:0:55822:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 0000000063dd8033/0x73cfd86cfc17b4fc lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x10e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03050114 expref: -99 pid: 55577 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1775764600.647967:0:55822:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764600.647976:0:55822:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x2d:0x0] updating i_size 16 i_blocks 1 00000020:00010000:1.0:1775764600.647980:0:55822:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1b230 (ffff8b2387cba518/0) at cl_lock_release():245 00000020:00010000:1.0:1775764600.647982:0:55822:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1b230 (ffff8b2387cba518/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764600.647985:0:55822:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318030 (ffff8b2387cba878/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764600.647988:0:55822:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1b230 (ffff8b2387cba518/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764600.647990:0:55822:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318030 (ffff8b2387cba518/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764600.648007:0:55822:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x2d:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764600.648028:0:55822:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764600.648033:0:55822:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764600.648040:0:55822:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5ea00/0x73cfd86cfc17b503 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050122 expref: -99 pid: 55816 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764600.648048:0:55822:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5ea00/0x73cfd86cfc17b503 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050122 expref: -99 pid: 55816 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764600.648055:0:55822:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764600.648058:0:55822:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764600.648061:0:55822:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b503 00000080:00010000:1.0:1775764600.648064:0:55822:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b503 from it ffffa60b01e33bf8 00010000:00010000:1.0:1775764600.648067:0:55822:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5ea00/0x73cfd86cfc17b503 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050122 expref: -99 pid: 55816 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764600.648079:0:55822:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5ea00/0x73cfd86cfc17b503 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050122 expref: -99 pid: 55816 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764600.648087:0:55822:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764600.648090:0:55822:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21d.sanity-pcc, flags=265 00000080:00200000:1.0:1775764600.648176:0:55822:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab1:0x2d:0x0](ffff8b23899c3248), inode mode 81a4 mask 44 00000080:00200008:1.0:1775764600.648183:0:55822:0:(file.c:1057:ll_file_open()) START file f21d.sanity-pcc:[0x200003ab1:0x2d:0x0](ffff8b23899c3248), flags 100000 00000002:00010000:1.0:1775764600.648197:0:55822:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x2d:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0240100001 00010000:00010000:1.0:1775764600.648254:0:55822:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741e00/0x73cfd86cfc17b50a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 55822 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764600.648261:0:55822:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741e00/0x73cfd86cfc17b50a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 55822 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764600.648274:0:55822:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764600.648290:0:55822:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fce00 x1862022098450688/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 928/132136 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764600.648300:0:55822:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741e00/0x73cfd86cfc17b50a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 55822 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764600.648324:0:55822:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:55822:1862022098450688:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764600.648664:0:55822:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85d4600 going to sleep for 11 seconds 00010000:00010000:1.0:1775764601.656282:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764604.731446:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764606.783700:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1775764607.745236:0:56292:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764607.745260:0:56292:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764607.745308:0:56292:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764607.745351:0:56292:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50400/0x73cfd86cfc17b53b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030501fb expref: -99 pid: 55844 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764607.745380:0:56292:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50400/0x73cfd86cfc17b53b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030501fb expref: -99 pid: 55844 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764607.745394:0:56292:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764607.745408:0:56292:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764607.745417:0:56292:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b53b 00000080:00010000:1.0:1775764607.745427:0:56292:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b53b from it ffffa60b0162bcb8 00010000:00010000:1.0:1775764607.745432:0:56292:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50400/0x73cfd86cfc17b53b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030501fb expref: -99 pid: 55844 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764607.745443:0:56292:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50400/0x73cfd86cfc17b53b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030501fb expref: -99 pid: 55844 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764607.805001:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764608.823639:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764613.496496:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764613.496504:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b557 lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03050287 expref: -99 pid: 56551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764613.496512:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5e200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764613.496528:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b557 lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03050287 expref: -99 pid: 56551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764613.496537:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b557 lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03050287 expref: -99 pid: 56551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764613.496551:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b557 lrc: 3/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03050287 expref: -99 pid: 56551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764613.496570:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8606000/0x73cfd86cfc17b0cd lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f273 expref: -99 pid: 51094 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764613.496600:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b557 lrc: 2/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03050287 expref: -99 pid: 56551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764613.496607:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764613.496625:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b557 lrc: 1/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03050287 expref: -99 pid: 56551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764613.496663:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b557 lrc: 0/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03050287 expref: -99 pid: 56551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764613.496676:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764613.496677:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764613.496679:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764613.496699:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098456960:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764613.508871:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098456960:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764613.597112:0:46140:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764613.597135:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119009664 00000100:00100000:1.0:1775764613.597148:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119009664 00000100:00100000:1.0:1775764613.597250:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 152 00000100:00100000:1.0:1775764613.597263:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23b9ff0c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42325:x1862022119009664:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764613.597286:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380031a00/0x73cfd86cfc17b550 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e0305026b expref: -99 pid: 56292 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764613.597372:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764613.597379:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23b9ff0c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42325:x1862022119009664:12345-192.168.202.128@tcp:104: Request processed in 120us (389us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764613.597386:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 152 00010000:00010000:1.0:1775764613.802906:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764613.802914:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b573 lrc: 1/0,0 mode: --/PR res: [0x200003ab1:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030502e9 expref: -99 pid: 56566 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764613.802923:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c50800 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764613.802928:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b573 lrc: 1/0,0 mode: --/PR res: [0x200003ab1:0x2f:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030502e9 expref: -99 pid: 56566 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764613.802935:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b573 lrc: 1/0,0 mode: --/PR res: [0x200003ab1:0x2f:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030502e9 expref: -99 pid: 56566 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764613.802943:0:46149:0:(ldlm_request.c:1765:ldlm_cli_cancel()) ### lock is already being destroyed 00010000:00010000:1.0:1775764613.802945:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b573 lrc: 1/0,0 mode: --/PR res: [0x200003ab1:0x2f:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030502e9 expref: -99 pid: 56566 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764613.802952:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b573 lrc: 0/0,0 mode: --/PR res: [0x200003ab1:0x2f:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030502e9 expref: -99 pid: 56566 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764613.802966:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764613.802968:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764613.802969:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764613.823226:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098458752:192.168.202.128@tcp:3:lfs.0 00000008:00010000:1.0:1775764613.827417:0:2376:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8b2384e75480 00000020:00200000:1.0:1775764613.827436:0:2376:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1775764613.827479:0:2376:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8b2384e75480 00000100:00100000:1.0:1775764613.827508:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098458752:192.168.202.128@tcp:3:lfs.0 00000080:00200000:1.0:1775764614.028046:0:56577:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764614.028082:0:56577:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764614.028106:0:56577:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764614.028125:0:56577:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764614.028133:0:56577:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764614.028143:0:56577:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764614.028146:0:56577:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b56c 00000080:00010000:1.0:1775764614.028153:0:56577:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b56c from it ffffa60b01e83bf8 00010000:00010000:1.0:1775764614.028157:0:56577:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764614.028167:0:56577:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764614.028182:0:56577:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764614.028199:0:56577:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f21e.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), file ffff8b238be89080, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764614.028205:0:56577:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f21e.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=open 00000002:00010000:1.0:1775764614.028220:0:56577:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f21e.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764614.028280:0:56577:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b581 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 56577 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764614.028288:0:56577:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b581 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 56577 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764614.028301:0:56577:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764614.028317:0:56577:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fea00 x1862022098459392/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 936/132136 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764614.028329:0:56577:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b581 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 56577 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764614.028357:0:56577:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:56577:1862022098459392:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764614.028595:0:56577:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23848b16c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764614.033390:0:56577:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:56577:1862022098459392:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764614.033415:0:56577:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b581 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 56577 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764614.033430:0:56577:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b581 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 56577 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764614.033437:0:56577:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b581 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 56577 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764614.033443:0:56577:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b581 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 56577 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764614.034982:0:56577:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b581 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 56577 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764614.035098:0:56577:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8740200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764614.035104:0:56577:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b581 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 56577 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764614.035111:0:56577:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b581 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 56577 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764614.035120:0:56577:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b581 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 56577 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764614.035128:0:56577:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764614.035136:0:56577:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b581 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 56577 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764614.035291:0:56577:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740200/0x73cfd86cfc17b581 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 56577 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764614.035328:0:56577:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8b23864fea00 x1862022098459392/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 936/608 e 0 to 0 dl 1775764630 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764614.035345:0:56577:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1775764614.036284:0:56577:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764614.036313:0:56577:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764614.036332:0:56577:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764614.036346:0:56577:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764614.036353:0:56577:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764614.036361:0:56577:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764614.036365:0:56577:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b56c 00000080:00010000:1.0:1775764614.036372:0:56577:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b56c from it ffffa60b01e83bb0 00010000:00010000:1.0:1775764614.036375:0:56577:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764614.036383:0:56577:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764614.036400:0:56577:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764614.036411:0:56577:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 304000 00000080:00200080:1.0:1775764614.037422:0:56577:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=c0086915 arg=1d023d0 00000100:00100000:1.0:1775764614.037711:0:56577:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:56577:1862022098459520:192.168.202.128@tcp:34:lfs.0 00000100:00100000:1.0:1775764614.037835:0:56577:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23848b16c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764614.042710:0:56577:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:56577:1862022098459520:192.168.202.128@tcp:34:lfs.0 00000080:00200000:1.0:1775764614.564667:0:56597:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764614.564699:0:56597:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764614.564722:0:56597:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 0 00010000:00010000:1.0:1775764614.564740:0:56597:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 0 00000080:00200000:1.0:1775764614.564747:0:56597:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764614.564758:0:56597:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764614.564762:0:56597:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b56c 00000080:00010000:1.0:1775764614.564769:0:56597:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b56c from it ffffa60b01ec3b58 00010000:00010000:1.0:1775764614.564774:0:56597:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 0 00010000:00010000:1.0:1775764614.564783:0:56597:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 0 00000080:00200000:1.0:1775764614.564800:0:56597:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764614.564815:0:56597:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=f21e.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags=8 00000080:00200000:1.0:1775764614.564818:0:56597:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f21e.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=getattr 00000002:00010000:1.0:1775764614.564832:0:56597:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f21e.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764614.564877:0:56597:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b5b9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 56597 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764614.564885:0:56597:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b5b9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 56597 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764614.564895:0:56597:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740a00/0x73cfd86cfc17b5b9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 56597 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764614.564924:0:56597:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job stat:7e6eda44-5508-487e-b17f-ff28ad9d3512:56597:1862022098461952:192.168.202.128@tcp:101:stat.0 00000100:00100000:1.0:1775764614.565026:0:56597:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23848b16c0 going to sleep for 11 seconds 00000080:00200008:1.0:1775764620.934768:0:56551:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 100000 00000100:00100000:1.0:1775764624.968115:0:2377:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2384b87800: committing for last_committed 34359738867 gen 1 00000100:00100000:1.0:1775764624.968135:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098463872:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764624.968157:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098464256:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764624.978893:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098463744:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764624.985979:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098464128:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764624.986001:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098464000:192.168.202.128@tcp:400:kworker.0 00000001:02000400:1.0:1775764626.215811:0:57196:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-pcc test 21f: mmap write should invalidate PCC-RO caching ========================================================== 15:57:04 (1775764624) 00000080:00200000:1.0:1775764628.296014:0:57297:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764628.296047:0:57297:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764628.296070:0:57297:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:1.0:1775764628.296089:0:57297:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:1.0:1775764628.296098:0:57297:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764628.296108:0:57297:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764628.296112:0:57297:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b56c 00000080:00010000:1.0:1775764628.296121:0:57297:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b56c from it ffffa60b01e03bb0 00010000:00010000:1.0:1775764628.296124:0:57297:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:1.0:1775764628.296134:0:57297:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:1.0:1775764628.296150:0:57297:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764628.296158:0:57297:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 100000 00000080:00200000:1.0:1775764628.296180:0:57297:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764628.296185:0:57297:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764628.296191:0:57297:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764628.296204:0:57297:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:1.0:1775764628.296210:0:57297:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764628.296214:0:57297:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764628.296216:0:57297:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b56c 00000080:00010000:1.0:1775764628.296219:0:57297:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b56c from it ffffa60b01e03bf8 00010000:00010000:1.0:1775764628.296222:0:57297:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764628.296229:0:57297:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b56c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030502c6 expref: -99 pid: 56560 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:1.0:1775764628.296236:0:57297:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764628.296241:0:57297:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=.lustre, flags=17 00000080:00200000:1.0:1775764628.296246:0:57297:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000002:0x1:0x0](ffff8b23899080c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764628.296257:0:57297:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=fid, dir=[0x200000002:0x1:0x0](ffff8b23899080c8), file ffff8b2386421680, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1775764628.296262:0:57297:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=fid, dir=[0x200000002:0x1:0x0](ffff8b23899080c8), intent=open 00000002:00010000:1.0:1775764628.296277:0:57297:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: fid,[0x0:0x0:0x0]) in obj [0x200000002:0x1:0x0], intent: open flags 0100001 00010000:00010000:1.0:1775764628.296328:0:57297:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b5c7 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 57297 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.296336:0:57297:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b5c7 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 57297 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764628.296349:0:57297:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764628.296368:0:57297:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fdf80 x1862022098464512/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 928/132136 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764628.296381:0:57297:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b5c7 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 57297 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764628.296410:0:57297:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:57297:1862022098464512:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764628.296801:0:57297:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2392d9e840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764628.324579:0:57297:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:57297:1862022098464512:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:1.0:1775764628.324609:0:57297:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b5c7 lrc: 4/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050439 expref: -99 pid: 57297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.324626:0:57297:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b5c7 lrc: 4/1,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050439 expref: -99 pid: 57297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.324633:0:57297:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b5c7 lrc: 4/1,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050439 expref: -99 pid: 57297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.324644:0:57297:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b5c7 lrc: 4/1,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050439 expref: -99 pid: 57297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.324654:0:57297:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b5c7 lrc: 4/1,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050439 expref: -99 pid: 57297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.324666:0:57297:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b5c7 lrc: 5/2,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050439 expref: -99 pid: 57297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.324675:0:57297:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b5c7 lrc: 5/2,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050439 expref: -99 pid: 57297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.324686:0:57297:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b5c7 lrc: 4/1,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050439 expref: -99 pid: 57297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764628.324697:0:57297:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23864fdf80 x1862022098464512/t34359738868(34359738868) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 928/4688 e 0 to 0 dl 1775764644 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764628.324710:0:57297:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fdf80 x1862022098464512/t34359738868(34359738868) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 928/4688 e 0 to 0 dl 1775764644 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764628.324719:0:57297:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b5c7 lrc: 3/1,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050439 expref: -99 pid: 57297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.324730:0:57297:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8606000/0x73cfd86cfc17b0cd lrc: 2/1,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f273 expref: -99 pid: 51094 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764628.324741:0:57297:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8606000/0x73cfd86cfc17b0cd lrc: 2/1,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f273 expref: -99 pid: 51094 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764628.324748:0:57297:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b5c7 lrc: 3/1,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050439 expref: -99 pid: 57297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.324754:0:57297:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b5c7 lrc: 3/1,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e03050439 expref: -99 pid: 57297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.324759:0:57297:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b5c7 lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e03050439 expref: -99 pid: 57297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.324797:0:57297:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:1.0:1775764628.324805:0:57297:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000002:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764628.324830:0:57297:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000002:0x2:0x0], updating i_size 6656 00000080:00200000:1.0:1775764628.324833:0:57297:0:(namei.c:147:ll_iget()) got inode: [0x200000002:0x2:0x0](ffff8b238990e3c8): rc = 0 00000080:00010000:1.0:1775764628.324837:0:57297:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000002:0x2:0x0](ffff8b238990e3c8) for lock 0x73cfd86cfc17b0cd 00000080:00010000:1.0:1775764628.324890:0:57297:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000002:0x2:0x0](ffff8b238990e3c8) 00000080:00200008:1.0:1775764628.324906:0:57297:0:(file.c:1057:ll_file_open()) START file fid:[0x200000002:0x2:0x0](ffff8b238990e3c8), flags 100000 00000002:00100000:1.0:1775764628.324924:0:57297:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fdf80 x1862022098464512/t34359738868(34359738868) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 928/4688 e 0 to 0 dl 1775764644 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1775764628.325566:0:57297:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b0cd from it ffff8b2389ed6780 00010000:00010000:1.0:1775764628.325574:0:57297:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8606000/0x73cfd86cfc17b0cd lrc: 3/1,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0304f273 expref: -99 pid: 51094 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764628.325587:0:57297:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8606000/0x73cfd86cfc17b0cd lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f273 expref: -99 pid: 51094 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 0 00000080:00200000:1.0:1775764628.325625:0:57297:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000002:0x2:0x0](ffff8b238990e3c8), inode mode 4040 mask 44 00000080:00200080:1.0:1775764628.325731:0:57297:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=401866d5 arg=1799980 00800000:00000080:1.0:1775764628.325748:0:57297:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=401866d5 len=28 karg=ffff8b23b8587fe0 uarg=0000000000000000 00000002:00000080:1.0:1775764628.325762:0:57297:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2384b87800: cmd=401866d5 len=28 karg=ffff8b23b8587fe0 uarg=0000000000000000 00000100:00100000:1.0:1775764628.325797:0:57297:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:57297:1862022098464768:192.168.202.128@tcp:59:lhsmtool_posix.0 00000100:00100000:1.0:1775764628.325907:0:57297:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2392d9e9c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764628.461630:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119012608 00000100:00100000:1.0:1775764628.461652:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119012608 00000100:00100000:1.0:1775764628.461738:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 153 00000100:00100000:1.0:1775764628.461749:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dad180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42325:x1862022119012608:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764628.461774:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380031600/0x73cfd86cfc17b5c0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e0305041d expref: -99 pid: 57037 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764628.461882:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764628.461889:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dad180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42325:x1862022119012608:12345-192.168.202.128@tcp:104: Request processed in 143us (3115us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764628.461898:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 153 00000100:00100000:1.0:1775764628.471061:0:57066:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:57066:1862022098465024:192.168.202.128@tcp:101:bash.0 00010000:00010000:1.0:1775764628.471087:0:57066:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b5ce lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305045c expref: -99 pid: 57066 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.471114:0:57066:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b5ce lrc: 4/0,1 mode: --/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305045c expref: -99 pid: 57066 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.471123:0:57066:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b5ce lrc: 4/0,1 mode: --/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305045c expref: -99 pid: 57066 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.471136:0:57066:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b5ce lrc: 4/0,1 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305045c expref: -99 pid: 57066 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.471146:0:57066:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b5ce lrc: 4/0,1 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305045c expref: -99 pid: 57066 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.471159:0:57066:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b5ce lrc: 5/1,1 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305045c expref: -99 pid: 57066 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.471167:0:57066:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b5ce lrc: 5/1,1 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305045c expref: -99 pid: 57066 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.471180:0:57066:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b5ce lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305045c expref: -99 pid: 57066 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764628.471191:0:57066:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8b2382dace00 x1862022098465024/t34359738869(34359738869) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 936/664 e 0 to 0 dl 1775764644 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764628.471204:0:57066:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382dace00 x1862022098465024/t34359738869(34359738869) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 936/664 e 0 to 0 dl 1775764644 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764628.471217:0:57066:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b5ce lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305045c expref: -99 pid: 57066 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764628.471229:0:57066:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b5ce lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305045c expref: -99 pid: 57066 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.471239:0:57066:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 4 res 8589949617/48 (0 0) 00000080:00200000:1.0:1775764628.471245:0:57066:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764628.471259:0:57066:0:(namei.c:1620:ll_create_it()) VFS Op:name=f21f.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=open|creat 00000080:00200000:1.0:1775764628.471281:0:57066:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab1:0x30:0x0](ffff8b2389a62a08) 00000080:00200000:1.0:1775764628.471396:0:57066:0:(file.c:6766:ll_layout_conf()) [0x200003ab1:0x30:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1775764628.471405:0:57066:0:(namei.c:1586:ll_create_node()) setting l_ast_data to inode [0x200003ab1:0x30:0x0](ffff8b2389a62a08) 00000080:00010000:1.0:1775764628.471409:0:57066:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x30:0x0](ffff8b2389a62a08) for lock 0x73cfd86cfc17b5ce 00000080:00010000:1.0:1775764628.471418:0:57066:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x30:0x0](ffff8b2389a62a08) 00000080:00200008:1.0:1775764628.471434:0:57066:0:(file.c:1057:ll_file_open()) START file f21f.sanity-pcc:[0x200003ab1:0x30:0x0](ffff8b2389a62a08), flags 101101 00000002:00100000:1.0:1775764628.471472:0:57066:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382dace00 x1862022098465024/t34359738869(34359738869) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764644 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1775764628.471518:0:57066:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b5ce from it ffff8b23849c34e0 00010000:00010000:1.0:1775764628.471523:0:57066:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b5ce lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305045c expref: -99 pid: 57066 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.471531:0:57066:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b5ce lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305045c expref: -99 pid: 57066 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764628.471572:0:57066:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab1:0x30:0x0](ffff8b2389a62a08), inode mode 81a4 mask 40 00000080:00200008:1.0:1775764628.471656:0:57066:0:(file.c:2612:do_file_write_iter()) START file f21f.sanity-pcc:[0x200003ab1:0x30:0x0], ppos: 0, count: 16 00000080:00200000:1.0:1775764628.471668:0:57066:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 16 bytes 00000080:00200000:1.0:1775764628.471674:0:57066:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 16 00000080:00200000:1.0:1775764628.471686:0:57066:0:(file.c:2058:ll_file_io_generic()) f21f.sanity-pcc: write ppos: 0, bytes: 16 00000080:00200000:1.0:1775764628.471691:0:57066:0:(file.c:1815:ll_io_set_mirror()) f21f.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764628.471698:0:57066:0:(cl_io.c:214:cl_io_rw_init()) header@00000000421a7a32[0x110, 1, [0x200003ab1:0x30:0x0] hash] 00000020:00200000:1.0:1775764628.471700:0:57066:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 16) 0 0 00000080:00200000:1.0:1775764628.471705:0:57066:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x30:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764628.471719:0:57066:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1775764628.471727:0:57066:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 16) 16, 16 00020000:00200000:1.0:1775764628.471731:0:57066:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764628.471759:0:57066:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1775764628.471772:0:57066:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764628.471777:0:57066:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318030, osc lock ffff8b2382cf6d10, flags 0x0 00010000:00010000:1.0:1775764628.471825:0:57066:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.471834:0:57066:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.471843:0:57066:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764628.471863:0:57066:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:57066:1862022098465280:192.168.202.128@tcp:101:bash.0 00000100:00100000:1.0:1775764628.471934:0:57066:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2392d9e9c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764628.477252:0:57066:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:57066:1862022098465280:192.168.202.128@tcp:101:bash.0 00010000:00010000:1.0:1775764628.477287:0:57066:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.477311:0:57066:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.477325:0:57066:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.477345:0:57066:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1775764628.477361:0:57066:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.477374:0:57066:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.477383:0:57066:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1775764628.477421:0:57066:0:(vvp_io.c:1286:vvp_io_write_start()) f21f.sanity-pcc: write [0, 16) append 0 append lockpos 0 00000080:00200000:1.0:1775764628.477448:0:57066:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 16 bytes 00000080:00200000:1.0:1775764628.477561:0:57066:0:(rw26.c:858:ll_write_end()) pos 0, len 16, copied 16 00000080:00200000:1.0:1775764628.477578:0:57066:0:(rw26.c:916:ll_write_end()) page@00000000c8013407[3 000000004f740785 2 1 00000000817b3741] 00000080:00200000:1.0:1775764628.477582:0:57066:0:(rw26.c:916:ll_write_end()) vmpage @000000003074839f 17ffffc0001001 3:0 ffff8b238a05f540 0 lru 00000080:00200000:1.0:1775764628.477591:0:57066:0:(rw26.c:916:ll_write_end()) osc-page@0000000034beaab6 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 000000004bfef949 00000000823c1ad6 > 3< 0 0 > 4< 0 0 8 23855104 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764628.477593:0:57066:0:(rw26.c:916:ll_write_end()) end page@00000000c8013407 00000080:00200000:1.0:1775764628.477595:0:57066:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764628.477602:0:57066:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 16 prio 0 00000008:00010000:1.0:1775764628.477631:0:57066:0:(osc_cache.c:848:osc_extent_find()) ### extent: 00000000545ea40f ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1775764628.477657:0:57066:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a62bb8, count 1, dirtied 1 00000080:00200000:1.0:1775764628.477663:0:57066:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 16 bytes, tot: 16 00000080:00200000:1.0:1775764628.477667:0:57066:0:(vvp_io.c:1353:vvp_io_write_start()) f21f.sanity-pcc: write bytes 0, result: 16 00000020:00010000:1.0:1775764628.477688:0:57066:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb7998 (ffff8b2387cba518/0) at cl_lock_release():245 00000020:00010000:1.0:1775764628.477691:0:57066:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb7998 (ffff8b2387cba518/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764628.477695:0:57066:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318030 (ffff8b2387cba878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764628.477700:0:57066:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764628.477710:0:57066:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:1.0:1775764628.477730:0:57066:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb7998 (ffff8b2387cba518/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764628.477733:0:57066:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318030 (ffff8b2387cba518/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764628.477742:0:57066:0:(vvp_io.c:517:vvp_io_advance()) advancing 16 bytes 00000080:00200000:1.0:1775764628.477752:0:57066:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:1.0:1775764628.477796:0:57066:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x30:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764628.477801:0:57066:0:(file.c:2206:ll_file_io_generic()) f21f.sanity-pcc: 2 io complete with rc: 0, result: 16, restart: 0 00000080:00200000:1.0:1775764628.477810:0:57066:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x30:0x0] updating i_size 16 i_blocks 1 00000080:00200000:1.0:1775764628.477816:0:57066:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 16 00000080:00200008:1.0:1775764628.477874:0:57066:0:(file.c:447:ll_file_release()) START file f21f.sanity-pcc:[0x200003ab1:0x30:0x0](ffff8b2389a62a08), flags 100001 00000002:00100000:1.0:1775764628.477946:0:57066:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b2382dace00 x1862022098465024/t34359738869(34359738869) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764644 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764628.477959:0:57066:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764628.477964:0:57066:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd500 x1862022098465536/t0(0) o35->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:23/10 lens 392/904 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764628.478005:0:57066:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:57066:1862022098465536:192.168.202.128@tcp:35:bash.0 00000100:00100000:1.0:1775764628.478121:0:57066:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2392d9e9c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764628.482251:0:57066:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:57066:1862022098465536:192.168.202.128@tcp:35:bash.0 00000002:00100000:1.0:1775764628.482291:0:57066:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b2382dace00 x1862022098465024/t34359738869(34359738869) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764644 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764628.612639:0:57312:0:(rw.c:1928:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1775764628.612699:0:57312:0:(file.c:2058:ll_file_io_generic()) f21f.sanity-pcc: read ppos: 0, bytes: 1048576 00000080:00200000:1.0:1775764628.612706:0:57312:0:(file.c:1815:ll_io_set_mirror()) f21f.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764628.612715:0:57312:0:(cl_io.c:214:cl_io_rw_init()) header@00000000421a7a32[0x110, 1, [0x200003ab1:0x30:0x0] hash] 00000020:00200000:1.0:1775764628.612717:0:57312:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 1048576) 0 0 00000080:00200000:1.0:1775764628.612722:0:57312:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x30:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764628.612744:0:57312:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab1:0x30:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764628.612976:0:57312:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:1.0:1775764628.612986:0:57312:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:1.0:1775764628.612990:0:57312:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764628.613024:0:57312:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:1.0:1775764628.613040:0:57312:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764628.613045:0:57312:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318030, osc lock ffff8b2382cf6d10, flags 0x400000000000000 00010000:00010000:1.0:1775764628.613071:0:57312:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764628.613088:0:57312:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764628.613101:0:57312:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:1.0:1775764628.613113:0:57312:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 16, setting rss=16; kms=16, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 5/1,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764628.613124:0:57312:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764628.613131:0:57312:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764628.613140:0:57312:0:(vvp_io.c:826:vvp_io_read_start()) f21f.sanity-pcc: read [0, 1048576) 00000001:00010000:1.0:1775764628.613156:0:57312:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab1:0x30:0x0] 00000008:00010000:1.0:1775764628.613162:0:57312:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764628.613164:0:57312:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318430, osc lock ffff8b2382cf6390, flags 0x400000400001002 00010000:00010000:1.0:1775764628.613171:0:57312:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764628.613181:0:57312:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1775764628.613190:0:57312:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764628.613192:0:57312:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764628.613204:0:57312:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 6/1,0 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1775764628.613210:0:57312:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764628.613216:0:57312:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x30:0x0] updating i_size 16 i_blocks 1 00000020:00010000:1.0:1775764628.613219:0:57312:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1b230 (ffff8b2387cba518/0) at cl_lock_release():245 00000020:00010000:1.0:1775764628.613222:0:57312:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1b230 (ffff8b2387cba518/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764628.613224:0:57312:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318430 (ffff8b2387cba878/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764628.613228:0:57312:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1b230 (ffff8b2387cba518/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764628.613230:0:57312:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318430 (ffff8b2387cba518/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764628.613240:0:57312:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:0 00000008:00010000:1.0:1775764628.613283:0:57312:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b238a0ce840 00000008:00010000:1.0:1775764628.613332:0:30878:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b238a0ce840 00010000:00010000:1.0:1775764628.613355:0:30878:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:1.0:1775764628.613732:0:30878:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23864fdc00 - object 0x280000400:271 offset 0<>4096 00000008:00010000:1.0:1775764628.623561:0:2376:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8b238a0ce840 00000020:00200000:1.0:1775764628.623579:0:2376:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1775764628.623601:0:2376:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8b238a0ce840 00000100:00100000:1.0:1775764628.623627:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098466560:192.168.202.128@tcp:3:lfs.0 00000020:00010000:1.0:1775764629.228216:0:57329:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384cd65d8 (ffff8b238a3ff2d8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764629.228223:0:57329:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384cd65d8 (ffff8b238a3ff2d8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764629.228227:0:57329:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238649beb0 (ffff8b238a3ff3f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764629.228238:0:57329:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764629.228261:0:57329:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000020:00010000:1.0:1775764629.228282:0:57329:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384cd65d8 (ffff8b238a3ff2d8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764629.228285:0:57329:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238649beb0 (ffff8b238a3ff2d8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764629.228342:0:57329:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x30:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764629.228565:0:57329:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x30:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764629.228579:0:57329:0:(lov_io.c:308:lov_io_mirror_write_intent()) [0x200003ab1:0x30:0x0]there are 0 components to be staled to modify file extent [0x0, 0xffffffffffffffff), iot: 5 00020000:00200000:1.0:1775764629.228585:0:57329:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab1:0x30:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764629.228588:0:57329:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1775764629.228596:0:57329:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764629.228623:0:57329:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1775764629.228642:0:57329:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764629.228646:0:57329:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318430, osc lock ffff8b2382cf6390, flags 0x0 00010000:00010000:1.0:1775764629.228669:0:57329:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764629.228684:0:57329:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764629.228696:0:57329:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764629.228704:0:57329:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764629.228717:0:57329:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000001:00010000:1.0:1775764629.228735:0:57329:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab1:0x30:0x0] 00000008:00010000:1.0:1775764629.228742:0:57329:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764629.228744:0:57329:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a3180b0, osc lock ffff8b2382cf7c80, flags 0x400001002 00010000:00010000:1.0:1775764629.228751:0:57329:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764629.228760:0:57329:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000020:00200000:1.0:1775764629.228769:0:57329:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764629.228772:0:57329:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 7/0,2 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764629.228780:0:57329:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000020:00200000:1.0:1775764629.228787:0:57329:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764629.228795:0:57329:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x30:0x0] updating i_size 16 i_blocks 1 00000020:00010000:1.0:1775764629.228799:0:57329:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f19da8 (ffff8b2387cba008/0) at cl_lock_release():245 00000020:00010000:1.0:1775764629.228801:0:57329:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f19da8 (ffff8b2387cba008/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764629.228804:0:57329:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a3180b0 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764629.228807:0:57329:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f19da8 (ffff8b2387cba008/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764629.228809:0:57329:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a3180b0 (ffff8b2387cba008/1) at cl_lock_fini():72 00000008:00010000:1.0:1775764629.228845:0:57329:0:(osc_cache.c:848:osc_extent_find()) ### extent: 000000003e2ad8bf ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000080:00200000:1.0:1775764629.228871:0:57329:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a62bb8, count 1, dirtied 1 00000020:00010000:1.0:1775764629.228891:0:57329:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb74e8 (ffff8b2387cba008/0) at cl_lock_release():245 00000020:00010000:1.0:1775764629.228893:0:57329:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb74e8 (ffff8b2387cba008/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764629.228896:0:57329:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318430 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764629.228900:0:57329:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764629.228921:0:57329:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00000020:00010000:1.0:1775764629.228932:0:57329:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb74e8 (ffff8b2387cba008/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764629.228934:0:57329:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318430 (ffff8b2387cba008/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764629.228954:0:57329:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x30:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1775764629.229074:0:57329:0:(file.c:447:ll_file_release()) START file f21f.sanity-pcc:[0x200003ab1:0x30:0x0](ffff8b2389a62a08), flags 100002 00000002:00100000:1.0:1775764629.229147:0:57329:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea7b80 x1862022098467584/t34359738874(34359738874) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764645 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764629.229160:0:57329:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764629.229165:0:57329:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fdc00 x1862022098468608/t0(0) o35->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:23/10 lens 392/904 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764629.229195:0:57329:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job multiop:7e6eda44-5508-487e-b17f-ff28ad9d3512:57329:1862022098468608:192.168.202.128@tcp:35:multiop.0 00000100:00100000:1.0:1775764629.229280:0:57329:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2392d9e9c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764629.251223:0:57329:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job multiop:7e6eda44-5508-487e-b17f-ff28ad9d3512:57329:1862022098468608:192.168.202.128@tcp:35:multiop.0 00000002:00100000:1.0:1775764629.251272:0:57329:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea7b80 x1862022098467584/t34359738874(34359738874) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764645 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764629.423446:0:57339:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:57339:1862022098468736:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764629.423542:0:57339:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.423559:0:57339:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.423576:0:57339:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.423591:0:57339:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.423609:0:57339:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.423620:0:57339:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.423628:0:57339:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764629.423642:0:57339:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b2382dad880 x1862022098468736/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764645 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764629.423664:0:57339:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.423688:0:57339:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1775764629.424274:0:57339:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764629.424324:0:57339:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:1.0:1775764629.424336:0:57339:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b60d 00000080:00010000:1.0:1775764629.424344:0:57339:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764629.424348:0:57339:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b60d from it ffffa60b01ec3bb0 00010000:00010000:1.0:1775764629.424353:0:57339:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.424366:0:57339:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764629.424408:0:57339:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764629.424430:0:57339:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 304000 00000080:00200080:1.0:1775764629.424631:0:57339:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=901066fc arg=7ffc008e3330 00000100:00100000:1.0:1775764629.424817:0:57339:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:57339:1862022098468864:192.168.202.128@tcp:34:lfs.0 00000100:00100000:1.0:1775764629.425076:0:57339:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2392d9e9c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764629.434733:0:57339:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:57339:1862022098468864:192.168.202.128@tcp:34:lfs.0 00000080:00200008:1.0:1775764629.435108:0:57339:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 304000 00000080:00200000:1.0:1775764629.483385:0:57340:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764629.483421:0:57340:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764629.483446:0:57340:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764629.483514:0:57340:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764629.483524:0:57340:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764629.483536:0:57340:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764629.483539:0:57340:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b60d 00000080:00010000:1.0:1775764629.483547:0:57340:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b60d from it ffffa60b01e4bbf8 00010000:00010000:1.0:1775764629.483551:0:57340:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764629.483562:0:57340:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764629.483587:0:57340:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764629.483610:0:57340:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f21f.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), file ffff8b2384e73e00, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764629.483619:0:57340:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f21f.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=open 00000002:00010000:1.0:1775764629.483641:0:57340:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f21f.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764629.483710:0:57340:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b614 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 57340 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.483720:0:57340:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b614 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 57340 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764629.483732:0:57340:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764629.483747:0:57340:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fdc00 x1862022098469120/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 936/132136 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764629.483763:0:57340:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b614 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 57340 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764629.483792:0:57340:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:57340:1862022098469120:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764629.574001:0:57347:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job mmap_cat:7e6eda44-5508-487e-b17f-ff28ad9d3512:57347:1862022098469376:192.168.202.128@tcp:101:mmap_cat.0 00010000:00010000:1.0:1775764629.574128:0:57347:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.574151:0:57347:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 4/1,0 mode: --/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.574160:0:57347:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 4/1,0 mode: --/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.574174:0:57347:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.574186:0:57347:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.574202:0:57347:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 5/2,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.574210:0:57347:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 5/2,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.574217:0:57347:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764629.574230:0:57347:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b2382daea00 x1862022098469376/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 584/840 e 0 to 0 dl 1775764645 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mmap_cat.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764629.574245:0:57347:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764629.574265:0:57347:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.574276:0:57347:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 4 res 8589949617/48 (0 0) 00000080:00200000:1.0:1775764629.574285:0:57347:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764629.574551:0:57347:0:(namei.c:147:ll_iget()) got inode: [0x200003ab1:0x30:0x0](ffff8b2389a62a08): rc = 0 00000080:00200000:1.0:1775764629.574576:0:57347:0:(file.c:6766:ll_layout_conf()) [0x200003ab1:0x30:0x0]: layout version change: 4 -> 4 00000080:00010000:1.0:1775764629.574588:0:57347:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x30:0x0](ffff8b2389a62a08) for lock 0x73cfd86cfc17b61b 00000080:00010000:1.0:1775764629.574604:0:57347:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x30:0x0](ffff8b2389a62a08) 00000080:00010000:1.0:1775764629.574608:0:57347:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b61b from it ffffa60b01edbb20 00010000:00010000:1.0:1775764629.574612:0:57347:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.574622:0:57347:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:1.0:1775764629.574671:0:57347:0:(file.c:6226:ll_getattr_dentry()) START file f21f.sanity-pcc:[0x200003ab1:0x30:0x0](ffff8b2389a62a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764629.574678:0:57347:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab1:0x30:0x0](ffff8b2389a62a08),name=f21f.sanity-pcc 00000002:00010000:1.0:1775764629.575121:0:57347:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x30:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764629.575136:0:57347:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764629.575150:0:57347:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764629.575157:0:57347:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764629.575165:0:57347:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x30:0x0](ffff8b2389a62a08) 00000080:00010000:1.0:1775764629.575168:0:57347:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x30:0x0](ffff8b2389a62a08) for lock 0x73cfd86cfc17b61b 00000080:00010000:1.0:1775764629.575172:0:57347:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b61b from it ffffa60b01edbcb8 00010000:00010000:1.0:1775764629.575175:0:57347:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764629.575182:0:57347:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764629.575232:0:57347:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x30:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1775764629.575246:0:57347:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab1:0x30:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764629.575250:0:57347:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1775764629.575255:0:57347:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab1:0x30:0x0] 00000008:00010000:1.0:1775764629.575294:0:57347:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764629.575298:0:57347:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318430, osc lock ffff8b2382cf6390, flags 0x400000400001002 00010000:00010000:1.0:1775764629.575324:0:57347:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764629.575337:0:57347:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00000020:00200000:1.0:1775764629.575353:0:57347:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764629.575359:0:57347:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764629.575546:0:57347:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00000020:00200000:1.0:1775764629.575562:0:57347:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764629.575579:0:57347:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x30:0x0] updating i_size 16 i_blocks 1 00000020:00010000:1.0:1775764629.575585:0:57347:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f19da8 (ffff8b2387cba008/0) at cl_lock_release():245 00000020:00010000:1.0:1775764629.575588:0:57347:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f19da8 (ffff8b2387cba008/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764629.575590:0:57347:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318430 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764629.575595:0:57347:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f19da8 (ffff8b2387cba008/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764629.575598:0:57347:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318430 (ffff8b2387cba008/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764629.575623:0:57347:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x30:0x0] ignore/verify layout 0/1, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764629.575843:0:57347:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764629.575858:0:57347:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764629.575869:0:57347:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764629.575879:0:57347:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764629.575885:0:57347:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764629.575891:0:57347:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764629.575894:0:57347:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b60d 00000080:00010000:1.0:1775764629.575898:0:57347:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b60d from it ffffa60b01edbbf8 00010000:00010000:1.0:1775764629.575902:0:57347:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764629.576199:0:57347:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764629.576213:0:57347:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764629.576226:0:57347:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21f.sanity-pcc, flags=265 00000080:00200000:1.0:1775764629.576234:0:57347:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab1:0x30:0x0](ffff8b2389a62a08), inode mode 81a4 mask 44 00000080:00200008:1.0:1775764629.576244:0:57347:0:(file.c:1057:ll_file_open()) START file f21f.sanity-pcc:[0x200003ab1:0x30:0x0](ffff8b2389a62a08), flags 100000 00000002:00010000:1.0:1775764629.576265:0:57347:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x30:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0240100001 00010000:00010000:1.0:1775764629.576343:0:57347:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b622 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 57347 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.576352:0:57347:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b622 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 57347 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764629.577008:0:57347:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764629.577212:0:57347:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd880 x1862022098469632/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 928/132136 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764629.577231:0:57347:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b622 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 57347 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764629.577272:0:57347:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job mmap_cat:7e6eda44-5508-487e-b17f-ff28ad9d3512:57347:1862022098469632:192.168.202.128@tcp:101:mmap_cat.0 00000100:00100000:1.0:1775764629.577756:0:57347:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2392d9e840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764629.581218:0:57347:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job mmap_cat:7e6eda44-5508-487e-b17f-ff28ad9d3512:57347:1862022098469632:192.168.202.128@tcp:101:mmap_cat.0 00010000:00010000:1.0:1775764629.581248:0:57347:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b622 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030505ac expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.581270:0:57347:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b622 lrc: 4/1,0 mode: --/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030505ac expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.581277:0:57347:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b622 lrc: 4/1,0 mode: --/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030505ac expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.581290:0:57347:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b622 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030505ac expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.581300:0:57347:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b622 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030505ac expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.581313:0:57347:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b622 lrc: 5/2,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030505ac expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.581321:0:57347:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b622 lrc: 5/2,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030505ac expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.581328:0:57347:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b622 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030505ac expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764629.581338:0:57347:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23864fd880 x1862022098469632/t34359738878(34359738878) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 928/864 e 0 to 0 dl 1775764645 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'mmap_cat.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764629.581350:0:57347:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fd880 x1862022098469632/t34359738878(34359738878) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 928/864 e 0 to 0 dl 1775764645 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'mmap_cat.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764629.581361:0:57347:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b622 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030505ac expref: -99 pid: 57347 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764629.581389:0:57347:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b622 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030505ac expref: -99 pid: 57347 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.581402:0:57347:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764629.581413:0:57347:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764629.581421:0:57347:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b622 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030505ac expref: -99 pid: 57347 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.581427:0:57347:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b622 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e030505ac expref: -99 pid: 57347 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.581434:0:57347:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740800/0x73cfd86cfc17b622 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e030505ac expref: -99 pid: 57347 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764629.581522:0:57347:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:1.0:1775764629.581531:0:57347:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764629.581662:0:57347:0:(file.c:6766:ll_layout_conf()) [0x200003ab1:0x30:0x0]: layout version change: 4 -> 4 00000080:00010000:1.0:1775764629.581674:0:57347:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x30:0x0](ffff8b2389a62a08) for lock 0x73cfd86cfc17b61b 00000080:00010000:1.0:1775764629.581682:0:57347:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b61b from it ffffa60b01edbbf0 00010000:00010000:1.0:1775764629.581685:0:57347:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764629.581978:0:57347:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c51800/0x73cfd86cfc17b61b lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305059e expref: -99 pid: 57347 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000002:00100000:1.0:1775764629.582013:0:57347:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fd880 x1862022098469632/t34359738878(34359738878) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764645 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'mmap_cat.0' uid:0 gid:0 projid:0 00000080:00a00000:1.0:1775764629.582137:0:57347:0:(llite_mmap.c:586:ll_file_mmap()) VFS_Op: fid=[0x200003ab1:0x30:0x0] vma=ffff8b238b64d658 start=0x7fbe11137000 end=0x7fbe11138000 vm_flags=0x71 00000080:00200000:1.0:1775764629.582163:0:57347:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x30:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1775764629.582174:0:57347:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab1:0x30:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764629.582178:0:57347:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1775764629.582182:0:57347:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab1:0x30:0x0] 00000008:00010000:1.0:1775764629.582214:0:57347:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764629.582218:0:57347:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318430, osc lock ffff8b2382cf6390, flags 0x400000400001002 00010000:00010000:1.0:1775764629.582238:0:57347:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00010000:00010000:1.0:1775764629.582251:0:57347:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00000020:00200000:1.0:1775764629.582263:0:57347:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764629.582267:0:57347:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00010000:00010000:1.0:1775764629.582274:0:57347:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000841d0f62/0x73cfd86cfc17b5d5 lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x10f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305046a expref: -99 pid: 57066 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00000020:00200000:1.0:1775764629.582284:0:57347:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764629.582293:0:57347:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x30:0x0] updating i_size 16 i_blocks 1 00000020:00010000:1.0:1775764629.582298:0:57347:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f19da8 (ffff8b2387cba008/0) at cl_lock_release():245 00000020:00010000:1.0:1775764629.582301:0:57347:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f19da8 (ffff8b2387cba008/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764629.582305:0:57347:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318430 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764629.582308:0:57347:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f19da8 (ffff8b2387cba008/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764629.582311:0:57347:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318430 (ffff8b2387cba008/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764629.582329:0:57347:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x30:0x0] ignore/verify layout 0/1, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1775764629.583111:0:57347:0:(file.c:447:ll_file_release()) START file f21f.sanity-pcc:[0x200003ab1:0x30:0x0](ffff8b2389a62a08), flags 100000 00000002:00100000:1.0:1775764629.589314:0:57347:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fd880 x1862022098469632/t34359738878(34359738878) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764645 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'mmap_cat.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764629.589328:0:57347:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764629.589332:0:57347:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd180 x1862022098470016/t0(0) o35->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:23/10 lens 392/904 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764629.589354:0:57347:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job mmap_cat:7e6eda44-5508-487e-b17f-ff28ad9d3512:57347:1862022098470016:192.168.202.128@tcp:35:mmap_cat.0 00000100:00100000:1.0:1775764629.589494:0:57347:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2392d9e840 going to sleep for 11 seconds 00000080:00200008:1.0:1775764636.110198:0:57770:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764636.110215:0:57770:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764636.110246:0:57770:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764636.110269:0:57770:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764636.110290:0:57770:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764636.110299:0:57770:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764636.110570:0:57770:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764636.110576:0:57770:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b60d 00000080:00010000:1.0:1775764636.110585:0:57770:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b60d from it ffffa60b0106fcb8 00010000:00010000:1.0:1775764636.110591:0:57770:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764636.110601:0:57770:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200008:1.0:1775764636.113434:0:57770:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764636.113446:0:57770:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764636.113559:0:57770:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764636.113584:0:57770:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764636.113605:0:57770:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764636.113614:0:57770:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764636.113626:0:57770:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764636.113630:0:57770:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b60d 00000080:00010000:1.0:1775764636.113736:0:57770:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b60d from it ffffa60b0106fcb8 00010000:00010000:1.0:1775764636.113740:0:57770:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764636.113750:0:57770:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764636.114286:0:57770:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764636.114298:0:57770:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764636.114620:0:57770:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764636.115088:0:57770:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764636.115100:0:57770:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764636.115107:0:57770:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764636.115110:0:57770:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b60d 00000080:00010000:1.0:1775764636.115118:0:57770:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b60d from it ffffa60b0106fbb0 00010000:00010000:1.0:1775764636.115121:0:57770:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764636.115129:0:57770:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764636.115142:0:57770:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764636.115151:0:57770:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 504000 00000080:00200080:1.0:1775764636.115179:0:57770:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=8080667f arg=7ffca4ece7b0 00000080:00200008:1.0:1775764636.115196:0:57770:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 504000 00000080:00200008:1.0:1775764636.629399:0:57787:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764636.629412:0:57787:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:1.0:1775764636.629565:0:57787:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764636.629801:0:57787:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741c00/0x73cfd86cfc17b629 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 57787 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764636.629813:0:57787:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741c00/0x73cfd86cfc17b629 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 57787 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764636.629825:0:57787:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741c00/0x73cfd86cfc17b629 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 57787 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764636.630156:0:57787:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc700 pname:cluuid:pid:xid:nid:opc:job lsof:82781e18-e0e3-4f9f-9a2b-041bf0344414:57787:1862022098471808:192.168.202.128@tcp:101:lsof.0 00000100:00100000:1.0:1775764636.630295:0:57787:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384904840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764640.314815:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098471936:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764640.314946:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098472192:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764640.314978:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098472448:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764640.315011:0:2377:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1775764640.315017:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098472064:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764640.315039:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098472320:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764640.318512:0:2377:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2384b87800: committing for last_committed 34359738880 gen 1 00000100:00100000:1.0:1775764640.318542:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098472064:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764640.318561:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098471936:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764640.318576:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098472448:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764640.319688:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098472320:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764640.319706:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098472192:192.168.202.128@tcp:400:kworker.0 00000080:00200008:1.0:1775764643.389415:0:58046:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764643.389438:0:58046:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764643.389699:0:58046:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764643.389719:0:58046:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1775764643.389741:0:58046:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:1.0:1775764643.389749:0:58046:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764643.389758:0:58046:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764643.389762:0:58046:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b60d 00000080:00010000:1.0:1775764643.389814:0:58046:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b60d from it ffffa60b01dd7cb8 00010000:00010000:1.0:1775764643.389819:0:58046:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1775764643.389829:0:58046:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200008:1.0:1775764643.391068:0:58046:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764643.391075:0:58046:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764643.391088:0:58046:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764643.391101:0:58046:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764643.391114:0:58046:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1775764643.391121:0:58046:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764643.391128:0:58046:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764643.391131:0:58046:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b60d 00000080:00010000:1.0:1775764643.391136:0:58046:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b60d from it ffffa60b01dd7cb8 00010000:00010000:1.0:1775764643.391140:0:58046:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764643.391148:0:58046:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764643.391269:0:58046:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764643.391276:0:58046:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764643.391283:0:58046:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764643.391291:0:58046:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764643.391393:0:58046:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764643.391398:0:58046:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764643.391400:0:58046:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b60d 00000080:00010000:1.0:1775764643.391403:0:58046:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b60d from it ffffa60b01dd7bb0 00010000:00010000:1.0:1775764643.391406:0:58046:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764643.391412:0:58046:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c50800/0x73cfd86cfc17b60d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050543 expref: -99 pid: 57339 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1775764643.391421:0:58046:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764643.391430:0:58046:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 100000 00000100:00100000:1.0:1775764643.598423:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764643.625293:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119016064 00000100:00100000:1.0:1775764643.625303:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119016064 00000100:00100000:1.0:1775764643.625444:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 155 00000100:00100000:1.0:1775764643.625490:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dafb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:44493:x1862022119016064:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764643.625641:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044a00/0x73cfd86cfc17b63e lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03050615 expref: -99 pid: 58056 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764643.625716:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764643.625723:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dafb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:44493:x1862022119016064:12345-192.168.202.128@tcp:104: Request processed in 271us (622us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764643.625730:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 155 00010000:00010000:1.0:1775764643.626818:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764643.626821:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1775764643.843862:0:58063:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764643.843896:0:58063:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764643.844012:0:58063:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741400/0x73cfd86cfc17b67d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 58063 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764643.844030:0:58063:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741400/0x73cfd86cfc17b67d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 58063 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764643.844042:0:58063:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741400/0x73cfd86cfc17b67d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 58063 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764643.844083:0:58063:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:58063:1862022098476416:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764643.844341:0:58063:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238a019780 going to sleep for 11 seconds 00010000:00010000:1.0:1775764643.970152:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764643.970163:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380031a00/0x73cfd86cfc17b68b lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e030506ee expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764643.970180:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2380031a00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764643.970186:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380031a00/0x73cfd86cfc17b68b lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x32:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e030506ee expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764643.970196:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380031a00/0x73cfd86cfc17b68b lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x32:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e030506ee expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764643.970206:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380031a00/0x73cfd86cfc17b68b lrc: 3/0,0 mode: PR/PR res: [0x200003ab1:0x32:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e030506ee expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764643.970315:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b676 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305069a expref: -99 pid: 58062 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764643.971014:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380031a00/0x73cfd86cfc17b68b lrc: 2/0,0 mode: --/PR res: [0x200003ab1:0x32:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030506ee expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764643.971025:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764643.971262:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380031a00/0x73cfd86cfc17b68b lrc: 1/0,0 mode: --/PR res: [0x200003ab1:0x32:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030506ee expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764643.971270:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380031a00/0x73cfd86cfc17b68b lrc: 0/0,0 mode: --/PR res: [0x200003ab1:0x32:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030506ee expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764643.971295:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764643.971297:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764643.971300:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764644.000774:0:58069:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:58069:1862022098477696:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764644.001002:0:58069:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 4/0,1 mode: --/EX res: [0x200003ab1:0x32:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764644.001016:0:58069:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 4/0,1 mode: --/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764644.001029:0:58069:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 4/0,1 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764644.001041:0:58069:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 4/0,1 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764644.001055:0:58069:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 5/1,1 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764644.001063:0:58069:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 5/1,1 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764644.001071:0:58069:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 4/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764644.001083:0:58069:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b2382daca80 x1862022098477696/t34359738892(34359738892) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 384/752 e 0 to 0 dl 1775764659 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764644.001108:0:58069:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 408 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764644.001120:0:58069:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764644.001135:0:58069:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 16 res 8589949617/50 (0 0) 00000080:00200000:1.0:1775764644.001141:0:58069:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab1:0x32:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1775764644.001150:0:58069:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x32:0x0](ffff8b23899c00c8) for lock 0x73cfd86cfc17b699 00000080:00010000:1.0:1775764644.001159:0:58069:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b699 from it ffffa60b01dcfc10 00010000:00010000:1.0:1775764644.001161:0:58069:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764644.001170:0:58069:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764644.001191:0:58069:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 2/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764644.001201:0:58069:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 2/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:1.0:1775764644.001209:0:58069:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab1:0x32:0x0](00000000ecd63baa) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764644.001298:0:58069:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764644.001305:0:58069:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764644.001335:0:58069:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x32:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764644.001358:0:58069:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab1:0x32:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764644.001362:0:58069:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1775764644.001572:0:58069:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:1.0:1775764644.001636:0:58069:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 0000000027d961df ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00010000:1.0:1775764644.001651:0:58069:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 0000000027d961df ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764644.002146:0:58069:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000008:00100000:1.0:1775764644.002239:0:58069:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23864ffb80 - object 0x240000400:272 offset 0<>22 00000080:00200000:1.0:1775764644.009980:0:58069:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x32:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1775764644.010104:0:58069:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 2/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764644.010121:0:58069:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 2/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1775764644.010133:0:58069:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab1:0x32:0x0](00000000ecd63baa) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764644.010210:0:58069:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x32:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764644.010221:0:58069:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab1:0x32:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764644.010224:0:58069:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1775764644.010230:0:58069:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764644.010381:0:58069:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x32:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1775764644.010400:0:58069:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 22 blocks 1 [cma]time 1775764643 1775764643 0 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000008:00010000:1.0:1775764644.010427:0:58069:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 0 blocks 1 [cma]time 1775764643 1775764643 0 ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000a7969a8b/0x73cfd86cfc17b66f lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e0305068c expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764644.010644:0:58069:0:(file.c:6766:ll_layout_conf()) [0x200003ab1:0x32:0x0]: layout version change: 4294967294 -> 4 00010000:00010000:1.0:1775764644.010656:0:58069:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764644.010666:0:58069:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045e00/0x73cfd86cfc17b699 lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050718 expref: -99 pid: 58069 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:1.0:1775764644.012005:0:58069:0:(file.c:2454:do_file_read_iter()) START file f21g.sanity-pcc:[0x200003ab1:0x32:0x0], ppos: 0, count: 1048576 00000080:00200000:1.0:1775764644.012042:0:58069:0:(rw.c:1928:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1775764644.012087:0:58069:0:(file.c:2058:ll_file_io_generic()) f21g.sanity-pcc: read ppos: 0, bytes: 1048576 00000080:00200000:1.0:1775764644.012094:0:58069:0:(file.c:1815:ll_io_set_mirror()) f21g.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764644.012102:0:58069:0:(cl_io.c:214:cl_io_rw_init()) header@0000000000e89250[0x110, 1, [0x200003ab1:0x32:0x0] hash] 00000020:00200000:1.0:1775764644.012105:0:58069:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 1048576) 0 0 00000080:00200000:1.0:1775764644.012110:0:58069:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x32:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764644.012126:0:58069:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab1:0x32:0x0]: flr state: 10, move mirror from 0 to 0, have retried: 0, mirror count: 3 00020000:00200000:1.0:1775764644.012130:0:58069:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:1.0:1775764644.012138:0:58069:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:1.0:1775764644.012142:0:58069:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764644.012175:0:58069:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:1.0:1775764644.012197:0:58069:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764644.012206:0:58069:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318830, osc lock ffff8b2382cf6390, flags 0x400000000000000 00010000:00010000:1.0:1775764644.012243:0:58069:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764644.012266:0:58069:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764644.012283:0:58069:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000008:00010000:1.0:1775764644.012296:0:58069:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 22, setting rss=22; kms=22, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764644.012307:0:58069:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764644.012315:0:58069:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764644.012326:0:58069:0:(vvp_io.c:826:vvp_io_read_start()) f21g.sanity-pcc: read [0, 1048576) 00000001:00010000:1.0:1775764644.012344:0:58069:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab1:0x32:0x0] 00000008:00010000:1.0:1775764644.012359:0:58069:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764644.012361:0:58069:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a0190f0, osc lock ffff8b2382cf7c80, flags 0x400000400001002 00000008:00010000:1.0:1775764644.012378:0:58069:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764644.012380:0:58069:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a019130, osc lock ffff8b2382cf7300, flags 0x400000400001002 00010000:00010000:1.0:1775764644.012388:0:58069:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764644.012397:0:58069:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1775764644.012407:0:58069:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764644.012410:0:58069:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764644.012416:0:58069:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764644.012427:0:58069:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000a7969a8b/0x73cfd86cfc17b66f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20080000000 nid: local remote: 0x57b8ff0e0305068c expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764644.012435:0:58069:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000a7969a8b/0x73cfd86cfc17b66f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20080000000 nid: local remote: 0x57b8ff0e0305068c expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:1.0:1775764644.012443:0:58069:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000a7969a8b/0x73cfd86cfc17b66f lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020080000000 nid: local remote: 0x57b8ff0e0305068c expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1775764644.012450:0:58069:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764644.012493:0:58069:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000a7969a8b/0x73cfd86cfc17b66f lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x57b8ff0e0305068c expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764644.012502:0:58069:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000a7969a8b/0x73cfd86cfc17b66f lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e0305068c expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1775764644.012518:0:58069:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764644.012526:0:58069:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x32:0x0] updating i_size 22 i_blocks 2 00000020:00010000:1.0:1775764644.012530:0:58069:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f19da8 (ffff8b2387cba008/0) at cl_lock_release():245 00000020:00010000:1.0:1775764644.012533:0:58069:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f19da8 (ffff8b2387cba008/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764644.012536:0:58069:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a0190f0 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764644.012540:0:58069:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a019130 (ffff8b2387cba638/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764644.012543:0:58069:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f19da8 (ffff8b2387cba008/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764644.012546:0:58069:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a0190f0 (ffff8b2387cba008/1) at cl_lock_fini():72 00000020:00010000:1.0:1775764644.012552:0:58069:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a019130 (ffff8b2387cba008/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764644.012560:0:58069:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:0 00000008:00010000:1.0:1775764644.012613:0:58069:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b238a0ce240 00000008:00010000:1.0:1775764644.012674:0:30878:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b238a0ce240 00010000:00010000:1.0:1775764644.013127:0:30878:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000008:00100000:1.0:1775764644.013172:0:30878:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23864fc380 - object 0x240000400:272 offset 0<>4096 00000080:00200000:1.0:1775764644.081414:0:57816:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764644.081471:0:57816:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764644.081496:0:57816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741400/0x73cfd86cfc17b67d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030506a8 expref: -99 pid: 58063 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764644.081514:0:57816:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741400/0x73cfd86cfc17b67d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030506a8 expref: -99 pid: 58063 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764644.081525:0:57816:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764644.081534:0:57816:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764644.081538:0:57816:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b67d 00000080:00010000:1.0:1775764644.081547:0:57816:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b67d from it ffffa60b01da3bf8 00010000:00010000:1.0:1775764644.081551:0:57816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741400/0x73cfd86cfc17b67d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030506a8 expref: -99 pid: 58063 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764644.081562:0:57816:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741400/0x73cfd86cfc17b67d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030506a8 expref: -99 pid: 58063 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764644.081580:0:57816:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764644.081599:0:57816:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764644.081605:0:57816:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764644.081610:0:57816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741400/0x73cfd86cfc17b67d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030506a8 expref: -99 pid: 58063 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764644.081618:0:57816:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741400/0x73cfd86cfc17b67d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030506a8 expref: -99 pid: 58063 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764644.081624:0:57816:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764644.081628:0:57816:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764644.081630:0:57816:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b67d 00000080:00010000:1.0:1775764644.081633:0:57816:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b67d from it ffffa60b01da3b38 00010000:00010000:1.0:1775764644.081635:0:57816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741400/0x73cfd86cfc17b67d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030506a8 expref: -99 pid: 58063 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764644.081642:0:57816:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741400/0x73cfd86cfc17b67d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030506a8 expref: -99 pid: 58063 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764644.081649:0:57816:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 3 00000080:00200000:1.0:1775764644.081654:0:57816:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f21g.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), file ffff8b2385297e00, open_flags 8241, mode 81b6 opened 0 00000080:00200000:1.0:1775764644.081664:0:57816:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f21g.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=open|creat 00000002:00010000:1.0:1775764644.081690:0:57816:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f21g.sanity-pcc,[0x200003ab1:0x34:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0101102 00010000:00010000:1.0:1775764644.081705:0:57816:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741400/0x73cfd86cfc17b67d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e030506a8 expref: -99 pid: 58063 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764644.081774:0:57816:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741400/0x73cfd86cfc17b67d lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030506a8 expref: -99 pid: 58063 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764644.081780:0:57816:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764644.081783:0:57816:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741400/0x73cfd86cfc17b67d lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030506a8 expref: -99 pid: 58063 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764644.082543:0:57816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741000/0x73cfd86cfc17b6a7 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 57816 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764644.082553:0:57816:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741000/0x73cfd86cfc17b6a7 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 57816 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764644.082566:0:57816:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764644.082581:0:57816:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fca80 x1862022098478848/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 936/66600 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764644.082590:0:57816:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741000/0x73cfd86cfc17b6a7 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 57816 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764644.082614:0:57816:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:57816:1862022098478848:192.168.202.128@tcp:101:bash.0 00000100:00100000:1.0:1775764644.082808:0:57816:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238a0190c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764644.129190:0:57816:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:57816:1862022098479488:192.168.202.128@tcp:101:bash.0 00010000:00010000:1.0:1775764644.129245:0:57816:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 4/0,1 mode: --/EX res: [0x200003ab1:0x32:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764644.129264:0:57816:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 4/0,1 mode: --/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764644.129279:0:57816:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 4/0,1 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764644.129291:0:57816:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 4/0,1 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764644.129311:0:57816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 5/1,1 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764644.129320:0:57816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 5/1,1 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764644.129328:0:57816:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 4/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764644.129342:0:57816:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea5500 x1862022098479488/t34359738897(34359738897) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 376/752 e 0 to 0 dl 1775764660 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764644.129384:0:57816:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 408 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764644.129405:0:57816:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764644.129422:0:57816:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 16 res 8589949617/50 (0 0) 00000080:00200000:1.0:1775764644.129432:0:57816:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab1:0x32:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1775764644.129444:0:57816:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x32:0x0](ffff8b23899c00c8) for lock 0x73cfd86cfc17b6b5 00000080:00010000:1.0:1775764644.129528:0:57816:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b6b5 from it ffffa60b01da3930 00010000:00010000:1.0:1775764644.129532:0:57816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764644.129543:0:57816:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764644.129574:0:57816:0:(file.c:1815:ll_io_set_mirror()) f21g.sanity-pcc: desiginated mirror: 0 00000080:00200000:1.0:1775764644.129587:0:57816:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x32:0x0] ignore/verify layout 0/0, layout version 4 restore needed 0 00010000:00010000:1.0:1775764644.129602:0:57816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 2/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764644.129615:0:57816:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 2/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:1.0:1775764644.129624:0:57816:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab1:0x32:0x0](00000000ecd63baa) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764644.129739:0:57816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764644.129748:0:57816:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764644.129782:0:57816:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x32:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764644.129797:0:57816:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab1:0x32:0x0]: flr state: 10, move mirror from 0 to 0, have retried: 0, mirror count: 3 00020000:00200000:1.0:1775764644.129801:0:57816:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1775764644.129809:0:57816:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764644.129917:0:57816:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x32:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1775764644.130043:0:57816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 2/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764644.130053:0:57816:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 2/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1775764644.130060:0:57816:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab1:0x32:0x0](00000000ecd63baa) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764644.130097:0:57816:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x32:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764644.130102:0:57816:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab1:0x32:0x0]: flr state: 10, move mirror from 0 to 0, have retried: 0, mirror count: 3 00020000:00200000:1.0:1775764644.130105:0:57816:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1775764644.130108:0:57816:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764644.130152:0:57816:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x32:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1775764644.130177:0:57816:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 22 blocks 1 [cma]time: 1775764643 1775764643 0 by oinfo size 22 blocks 1 [cma]time 1775764643 1775764643 1775764644 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000008:00010000:1.0:1775764644.130214:0:57816:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 1775764643 1775764643 0 by oinfo size 0 blocks 1 [cma]time 1775764643 1775764643 0 ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000a7969a8b/0x73cfd86cfc17b66f lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e0305068c expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764644.130291:0:57816:0:(file.c:6766:ll_layout_conf()) [0x200003ab1:0x32:0x0]: layout version change: 4294967294 -> 5 00010000:00010000:1.0:1775764644.130302:0:57816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764644.130311:0:57816:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b2380030200/0x73cfd86cfc17b6b5 lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x32:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050781 expref: -99 pid: 57816 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00020000:00200000:1.0:1775764644.130333:0:57816:0:(lov_io.c:308:lov_io_mirror_write_intent()) [0x200003ab1:0x32:0x0]there are 0 components to be staled to modify file extent [0x0, 0xffffffffffffffff), iot: 3 00020000:00200000:1.0:1775764644.130337:0:57816:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab1:0x32:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 3 00020000:00200000:1.0:1775764644.130340:0:57816:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1775764644.130360:0:57816:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764644.130387:0:57816:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1775764644.130403:0:57816:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 4 00000008:00010000:1.0:1775764644.130410:0:57816:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a019ab0, osc lock ffff8b2382cf7300, flags 0x80000000 00000008:00010000:1.0:1775764644.130414:0:57816:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 4 00000008:00010000:1.0:1775764644.130415:0:57816:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a019af0, osc lock ffff8b2382cf7c80, flags 0x80000000 00010000:00010000:1.0:1775764644.130430:0:57816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764644.130443:0:57816:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764644.130492:0:57816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000008:00010000:1.0:1775764644.130503:0:57816:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 22, setting rss=22; kms=22, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764644.130515:0:57816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764644.130521:0:57816:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764644.130534:0:57816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000a7969a8b/0x73cfd86cfc17b66f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20080000000 nid: local remote: 0x57b8ff0e0305068c expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764644.130544:0:57816:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000a7969a8b/0x73cfd86cfc17b66f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20080000000 nid: local remote: 0x57b8ff0e0305068c expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764644.130552:0:57816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000a7969a8b/0x73cfd86cfc17b66f lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020080000000 nid: local remote: 0x57b8ff0e0305068c expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000008:00010000:1.0:1775764644.130559:0:57816:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000a7969a8b/0x73cfd86cfc17b66f lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020080000000 nid: local remote: 0x57b8ff0e0305068c expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764644.130567:0:57816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000a7969a8b/0x73cfd86cfc17b66f lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x57b8ff0e0305068c expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764644.130574:0:57816:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000a7969a8b/0x73cfd86cfc17b66f lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x57b8ff0e0305068c expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:1.0:1775764644.130696:0:2377:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764644.130721:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098479616:192.168.202.128@tcp:10:bash.0 00000100:00100000:1.0:1775764644.131066:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098479744:192.168.202.128@tcp:10:bash.0 00000080:00200000:1.0:1775764644.140986:0:57816:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x32:0x0] updating i_size 0 i_blocks 2 00000020:00010000:1.0:1775764644.140994:0:57816:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b4e8 (ffff8b23812ece18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764644.140998:0:57816:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b4e8 (ffff8b23812ece18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764644.141004:0:57816:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a019ab0 (ffff8b2387cba5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764644.141015:0:57816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764644.141032:0:57816:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00010000:1.0:1775764644.141051:0:57816:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a019af0 (ffff8b2387cba758/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764644.141057:0:57816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000a7969a8b/0x73cfd86cfc17b66f lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x57b8ff0e0305068c expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764644.141064:0:57816:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000a7969a8b/0x73cfd86cfc17b66f lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e0305068c expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00010000:1.0:1775764644.141072:0:57816:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b4e8 (ffff8b23812ece18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764644.141075:0:57816:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a019ab0 (ffff8b23812ece18/1) at cl_lock_fini():72 00000020:00010000:1.0:1775764644.141081:0:57816:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a019af0 (ffff8b23812ece18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764644.141140:0:57816:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x32:0x0] ignore/verify layout 0/0, layout version 5 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1775764644.141172:0:57816:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file f21g.sanity-pcc:[0x200003ab1:0x32:0x0](ffff8b23899c00c8) current size 0, valid attrs a168, mode 0, uid 2, gid 0, new size 0, atime 1775764644.000000000, mtime 1775764644.000000000, ctime 1775764644.000000000, rc 0 00000080:00200008:1.0:1775764644.141933:0:57816:0:(file.c:2612:do_file_write_iter()) START file f21g.sanity-pcc:[0x200003ab1:0x32:0x0], ppos: 0, count: 29 00000080:00200000:1.0:1775764644.141955:0:57816:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 29 bytes 00000080:00200000:1.0:1775764644.141962:0:57816:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 29 00000080:00200000:1.0:1775764644.141994:0:57816:0:(file.c:2058:ll_file_io_generic()) f21g.sanity-pcc: write ppos: 0, bytes: 29 00000080:00200000:1.0:1775764644.142000:0:57816:0:(file.c:1815:ll_io_set_mirror()) f21g.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764644.142008:0:57816:0:(cl_io.c:214:cl_io_rw_init()) header@0000000000e89250[0x110, 1, [0x200003ab1:0x32:0x0] hash] 00000020:00200000:1.0:1775764644.142011:0:57816:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 29) 0 0 00000080:00200000:1.0:1775764644.142015:0:57816:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x32:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764644.142030:0:57816:0:(lov_io.c:308:lov_io_mirror_write_intent()) [0x200003ab1:0x32:0x0]there are 0 components to be staled to modify file extent [0x0, 0xffffffffffffffff), iot: 2 00020000:00200000:1.0:1775764644.142036:0:57816:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab1:0x32:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 3 00020000:00200000:1.0:1775764644.142038:0:57816:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:1.0:1775764644.142043:0:57816:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1775764644.142052:0:57816:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 29) 29, 29 00020000:00200000:1.0:1775764644.142055:0:57816:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764644.142086:0:57816:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1775764644.142101:0:57816:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764644.142107:0:57816:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318fb0, osc lock ffff8b2382cf7c80, flags 0x0 00010000:00010000:1.0:1775764644.142132:0:57816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764644.142147:0:57816:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764644.142159:0:57816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764644.142167:0:57816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764644.142175:0:57816:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764644.142187:0:57816:0:(vvp_io.c:1286:vvp_io_write_start()) f21g.sanity-pcc: write [0, 29) append 0 append lockpos 0 00000080:00200000:1.0:1775764644.142196:0:57816:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 29 bytes 00000080:00200000:1.0:1775764644.142979:0:57816:0:(rw26.c:858:ll_write_end()) pos 0, len 29, copied 29 00000080:00200000:1.0:1775764644.142995:0:57816:0:(rw26.c:916:ll_write_end()) page@0000000094e1169f[3 00000000df6eb339 2 1 000000007cfbe91e] 00000080:00200000:1.0:1775764644.142999:0:57816:0:(rw26.c:916:ll_write_end()) vmpage @000000002ac13c84 17ffffc0001001 3:0 ffff8b238a05f7e0 0 lru 00000080:00200000:1.0:1775764644.143009:0:57816:0:(rw26.c:916:ll_write_end()) osc-page@000000008a45ed9b 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 00000000bf12bc0b 00000000823c1ad6 > 3< 0 0 > 4< 0 0 8 20447232 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764644.143011:0:57816:0:(rw26.c:916:ll_write_end()) end page@0000000094e1169f 00000080:00200000:1.0:1775764644.143013:0:57816:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764644.143020:0:57816:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 29 prio 0 00000008:00010000:1.0:1775764644.143052:0:57816:0:(osc_cache.c:848:osc_extent_find()) ### extent: 00000000f6451404 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764644.143082:0:57816:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b23899c0278, count 1, dirtied 1 00000080:00200000:1.0:1775764644.143088:0:57816:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 29 bytes, tot: 29 00000080:00200000:1.0:1775764644.143093:0:57816:0:(vvp_io.c:1353:vvp_io_write_start()) f21g.sanity-pcc: write bytes 0, result: 29 00000020:00010000:1.0:1775764644.143115:0:57816:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb7b78 (ffff8b2387cba758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764644.143119:0:57816:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb7b78 (ffff8b2387cba758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764644.143122:0:57816:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318fb0 (ffff8b2387cba5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764644.143129:0:57816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764644.143140:0:57816:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00000020:00010000:1.0:1775764644.143157:0:57816:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb7b78 (ffff8b2387cba758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764644.143161:0:57816:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318fb0 (ffff8b2387cba758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764644.143170:0:57816:0:(vvp_io.c:517:vvp_io_advance()) advancing 29 bytes 00000080:00200000:1.0:1775764644.143176:0:57816:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:1.0:1775764644.143207:0:57816:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x32:0x0] ignore/verify layout 0/0, layout version 5 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764644.143212:0:57816:0:(file.c:2206:ll_file_io_generic()) f21g.sanity-pcc: 2 io complete with rc: 0, result: 29, restart: 0 00000080:00200000:1.0:1775764644.143224:0:57816:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x32:0x0] updating i_size 29 i_blocks 2 00000080:00200000:1.0:1775764644.144370:0:57816:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 29 00000080:00200008:1.0:1775764644.144472:0:57816:0:(file.c:447:ll_file_release()) START file f21g.sanity-pcc:[0x200003ab1:0x32:0x0](ffff8b23899c00c8), flags 100001 00000002:00100000:1.0:1775764644.144591:0:57816:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fca80 x1862022098478848/t34359738894(34359738894) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 936/1016 e 0 to 0 dl 1775764660 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764644.144608:0:57816:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764644.144614:0:57816:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fed80 x1862022098480000/t0(0) o35->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:23/10 lens 392/904 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764644.144651:0:57816:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:57816:1862022098480000:192.168.202.128@tcp:35:bash.0 00000100:00100000:1.0:1775764644.146130:0:57816:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238a019a80 going to sleep for 11 seconds 00000100:00100000:1.0:1775764645.431900:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098480768:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764645.432021:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098481024:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764645.432062:0:2377:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:1.0:1775764645.432069:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098480640:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764645.432096:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098480896:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764645.432117:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098481152:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764645.450704:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098480640:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764645.450734:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098480896:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764645.450747:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098481152:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764645.450760:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098481024:192.168.202.128@tcp:400:kworker.0 00010000:00010000:1.0:1775764654.904709:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764655.673731:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098482432:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764655.673767:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098482688:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764655.673785:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098482944:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764655.679910:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098482688:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764655.679929:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098482944:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764655.681382:0:2376:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2384b87800: committing for last_committed 34359738899 gen 1 00000100:00100000:1.0:1775764655.681402:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098482560:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764655.681420:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098482816:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764655.681977:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098482432:192.168.202.128@tcp:400:kworker.0 00010000:00010000:1.0:1775764656.958301:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1775764658.310358:0:58773:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764658.310391:0:58773:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764658.310418:0:58773:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045a00/0x73cfd86cfc17b6bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305078f expref: -99 pid: 58071 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1775764658.310439:0:58773:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045a00/0x73cfd86cfc17b6bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305078f expref: -99 pid: 58071 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:1.0:1775764658.310448:0:58773:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764658.310490:0:58773:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764658.310494:0:58773:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b6bc 00000080:00010000:1.0:1775764658.310502:0:58773:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b6bc from it ffffa60b016cbbb0 00010000:00010000:1.0:1775764658.310506:0:58773:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045a00/0x73cfd86cfc17b6bc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305078f expref: -99 pid: 58071 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1775764658.310516:0:58773:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045a00/0x73cfd86cfc17b6bc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305078f expref: -99 pid: 58071 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1775764658.310531:0:58773:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764658.310539:0:58773:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 100000 00000080:00200000:1.0:1775764658.310563:0:58773:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764658.310572:0:58773:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764658.310579:0:58773:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045a00/0x73cfd86cfc17b6bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305078f expref: -99 pid: 58071 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764658.310587:0:58773:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045a00/0x73cfd86cfc17b6bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305078f expref: -99 pid: 58071 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1775764658.310594:0:58773:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764658.310598:0:58773:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764658.310600:0:58773:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b6bc 00000080:00010000:1.0:1775764658.310603:0:58773:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b6bc from it ffffa60b016cbbf8 00010000:00010000:1.0:1775764658.310606:0:58773:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045a00/0x73cfd86cfc17b6bc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305078f expref: -99 pid: 58071 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764658.310614:0:58773:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045a00/0x73cfd86cfc17b6bc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305078f expref: -99 pid: 58071 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764658.310622:0:58773:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764658.310628:0:58773:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=.lustre, flags=17 00000080:00200000:1.0:1775764658.310633:0:58773:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000002:0x1:0x0](ffff8b23899080c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764658.310644:0:58773:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=fid, dir=[0x200000002:0x1:0x0](ffff8b23899080c8), file ffff8b2383ef0480, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1775764658.310649:0:58773:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=fid, dir=[0x200000002:0x1:0x0](ffff8b23899080c8), intent=open 00000002:00010000:1.0:1775764658.310665:0:58773:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: fid,[0x0:0x0:0x0]) in obj [0x200000002:0x1:0x0], intent: open flags 0100001 00010000:00010000:1.0:1775764658.310718:0:58773:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b6d1 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 58773 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764658.310726:0:58773:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b6d1 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 58773 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764658.310738:0:58773:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764658.311288:0:58773:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd180 x1862022098483200/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 928/132136 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764658.311301:0:58773:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b6d1 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 58773 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764658.311331:0:58773:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:58773:1862022098483200:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764658.312046:0:58773:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b8759540 going to sleep for 11 seconds 00010000:00010000:1.0:1775764658.321211:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764658.321230:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b6d1 lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e030507e3 expref: -99 pid: 58773 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764658.321241:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8741800 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764658.321247:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b6d1 lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e030507e3 expref: -99 pid: 58773 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764658.321256:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b6d1 lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e030507e3 expref: -99 pid: 58773 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764658.321266:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b6d1 lrc: 3/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e030507e3 expref: -99 pid: 58773 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764658.321290:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8606000/0x73cfd86cfc17b0cd lrc: 3/1,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f273 expref: -99 pid: 51094 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764658.321343:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b6d1 lrc: 2/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030507e3 expref: -99 pid: 58773 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764658.321350:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764658.321380:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b6d1 lrc: 1/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030507e3 expref: -99 pid: 58773 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764658.321387:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b6d1 lrc: 0/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030507e3 expref: -99 pid: 58773 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764658.321404:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764658.321406:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764658.321407:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764658.321427:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098483328:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764658.327594:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098483328:192.168.202.128@tcp:103:ldlm_bl.0 00000080:00200000:1.0:1775764658.381003:0:58781:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764658.381036:0:58781:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764658.381058:0:58781:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045a00/0x73cfd86cfc17b6bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305078f expref: -99 pid: 58071 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764658.381075:0:58781:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045a00/0x73cfd86cfc17b6bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305078f expref: -99 pid: 58071 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764658.381087:0:58781:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764658.381096:0:58781:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764658.381112:0:58781:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b6bc 00000080:00010000:1.0:1775764658.381120:0:58781:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b6bc from it ffffa60b01e4bbb0 00010000:00010000:1.0:1775764658.381124:0:58781:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045a00/0x73cfd86cfc17b6bc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305078f expref: -99 pid: 58071 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764658.381133:0:58781:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045a00/0x73cfd86cfc17b6bc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305078f expref: -99 pid: 58071 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1775764658.381149:0:58781:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764658.381158:0:58781:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 504000 00000080:00200080:1.0:1775764658.381186:0:58781:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=8080667f arg=7ffcfb2ed870 00000080:00200008:1.0:1775764658.381201:0:58781:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 504000 00000080:00200000:1.0:1775764658.400093:0:58782:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764658.400222:0:58782:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764658.400248:0:58782:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045a00/0x73cfd86cfc17b6bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305078f expref: -99 pid: 58071 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764658.400268:0:58782:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045a00/0x73cfd86cfc17b6bc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305078f expref: -99 pid: 58071 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1775764658.400276:0:58782:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764658.400284:0:58782:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764658.400288:0:58782:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b6bc 00000080:00010000:1.0:1775764658.400296:0:58782:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b6bc from it ffffa60b0146bb58 00010000:00010000:1.0:1775764658.400301:0:58782:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045a00/0x73cfd86cfc17b6bc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305078f expref: -99 pid: 58071 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764658.400310:0:58782:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2045a00/0x73cfd86cfc17b6bc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305078f expref: -99 pid: 58071 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:1.0:1775764658.400328:0:58782:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764658.400342:0:58782:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=f21h.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags=13 00000080:00200000:1.0:1775764658.400346:0:58782:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f21h.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=getattr 00000002:00010000:1.0:1775764658.400360:0:58782:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f21h.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764658.400417:0:58782:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b6d8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 58782 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764658.400425:0:58782:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b6d8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 58782 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764658.400436:0:58782:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b6d8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 58782 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764658.400616:0:58782:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:58782:1862022098483584:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764658.422338:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764658.451115:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:1.0:1775764659.002583:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1775764659.010998:0:58801:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764659.011026:0:58801:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764659.011044:0:58801:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b6ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050845 expref: -99 pid: 58783 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764659.011061:0:58801:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b6ed lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050845 expref: -99 pid: 58783 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764659.011071:0:58801:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764659.011080:0:58801:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764659.011084:0:58801:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b6ed 00000080:00010000:1.0:1775764659.011091:0:58801:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b6ed from it ffffa60b01ea3bf8 00010000:00010000:1.0:1775764659.011172:0:58801:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b6ed lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050845 expref: -99 pid: 58783 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764659.011183:0:58801:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b6ed lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050845 expref: -99 pid: 58783 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764659.011199:0:58801:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764659.011224:0:58801:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f21h.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), file ffff8b23a00a6f00, open_flags 4d902, mode 0 opened 0 00000080:00200000:1.0:1775764659.011232:0:58801:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f21h.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=open 00000002:00010000:1.0:1775764659.011246:0:58801:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f21h.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 01154403 00010000:00010000:1.0:1775764659.011387:0:58801:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741c00/0x73cfd86cfc17b725 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 58801 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.011395:0:58801:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741c00/0x73cfd86cfc17b725 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 58801 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764659.011409:0:58801:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764659.011421:0:58801:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fdf80 x1862022098487424/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 936/132136 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764659.011430:0:58801:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741c00/0x73cfd86cfc17b725 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 58801 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764659.011522:0:58801:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:58801:1862022098487424:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764659.011819:0:58801:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b8759000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764659.035287:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764659.037041:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098488064:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764659.278680:0:46140:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764659.278697:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119020160 00000100:00100000:1.0:1775764659.278706:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119020160 00000100:00100000:1.0:1775764659.278756:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 159 00000100:00100000:1.0:1775764659.278764:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2389470c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42946:x1862022119020160:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764659.278785:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000af481ba1/0x73cfd86cfc17b741 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x114:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e03050964 expref: -99 pid: 58801 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764659.278864:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764659.278870:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2389470c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42946:x1862022119020160:12345-192.168.202.128@tcp:104: Request processed in 106us (274us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764659.278877:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 159 00000100:00100000:1.0:1775764659.282254:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764659.287800:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119020416 00000100:00100000:1.0:1775764659.287811:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119020416 00000100:00100000:1.0:1775764659.287874:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 161 00000100:00100000:1.0:1775764659.287885:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daf480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42944:x1862022119020416:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764659.287904:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 00000000887119ad/0x73cfd86cfc17b748 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x114:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e0305096b expref: -99 pid: 58801 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764659.287986:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764659.287993:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daf480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42944:x1862022119020416:12345-192.168.202.128@tcp:104: Request processed in 110us (555us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764659.288005:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 161 00000100:00100000:1.0:1775764659.294915:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119020544 00000100:00100000:1.0:1775764659.294926:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119020544 00000100:00100000:1.0:1775764659.294983:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 162 00000100:00100000:1.0:1775764659.294994:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dae680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42310:x1862022119020544:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764659.295011:0:46140:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000007eca9ddf/0x73cfd86cfc17b764 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e030509b8 expref: -99 pid: 58801 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.295025:0:46140:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000007eca9ddf/0x73cfd86cfc17b764 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e030509b8 expref: -99 pid: 58801 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.295033:0:46140:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000007eca9ddf/0x73cfd86cfc17b764 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e030509b8 expref: -99 pid: 58801 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.295043:0:46140:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000007eca9ddf/0x73cfd86cfc17b764 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000020000 nid: local remote: 0x57b8ff0e030509b8 expref: -99 pid: 58801 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.295053:0:46140:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000007eca9ddf/0x73cfd86cfc17b764 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000020000 nid: local remote: 0x57b8ff0e030509b8 expref: -99 pid: 58801 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.295063:0:46140:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000007eca9ddf/0x73cfd86cfc17b764 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80000020000 nid: local remote: 0x57b8ff0e030509b8 expref: -99 pid: 58801 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.295101:0:46140:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b23ac6ea000) 00000100:00100000:1.0:1775764659.295162:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dae680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42310:x1862022119020544:12345-192.168.202.128@tcp:105: Request processed in 170us (806us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764659.295170:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 162 00000100:00100000:1.0:1775764659.311701:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764659.315798:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764659.321646:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764659.325318:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764659.340688:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764659.346430:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764659.371286:0:2368:0:(events.c:108:reply_in_callback()) truncate req ffff8b23b9ea5880 rpc 752 - 880+224 00010000:00010000:1.0:1775764659.678223:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764659.678235:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb000/0x73cfd86cfc17b79c lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03050a7c expref: -99 pid: 58819 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.678247:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6eb000 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764659.678251:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb000/0x73cfd86cfc17b79c lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x35:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03050a7c expref: -99 pid: 58819 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.678260:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb000/0x73cfd86cfc17b79c lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x35:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03050a7c expref: -99 pid: 58819 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.678269:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb000/0x73cfd86cfc17b79c lrc: 3/0,0 mode: PR/PR res: [0x200003ab1:0x35:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03050a7c expref: -99 pid: 58819 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.678294:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044600/0x73cfd86cfc17b795 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050a6e expref: -99 pid: 58819 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764659.678342:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb000/0x73cfd86cfc17b79c lrc: 2/0,0 mode: --/PR res: [0x200003ab1:0x35:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03050a7c expref: -99 pid: 58819 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.678354:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764659.678405:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb000/0x73cfd86cfc17b79c lrc: 1/0,0 mode: --/PR res: [0x200003ab1:0x35:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03050a7c expref: -99 pid: 58819 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.678412:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb000/0x73cfd86cfc17b79c lrc: 0/0,0 mode: --/PR res: [0x200003ab1:0x35:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03050a7c expref: -99 pid: 58819 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.678425:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764659.678427:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764659.678428:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764659.678438:0:2377:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764659.678492:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098494720:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764659.684007:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098494720:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764659.690081:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119022464 00000100:00100000:1.0:1775764659.690105:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119022464 00000100:00100000:1.0:1775764659.690169:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 169 00000100:00100000:1.0:1775764659.690191:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dae680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43212:x1862022119022464:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764659.690220:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000ffecd843/0x73cfd86cfc17b75d lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x114:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000020000 nid: local remote: 0x57b8ff0e030509b1 expref: -99 pid: 58801 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764659.690314:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764659.690320:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dae680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43212:x1862022119022464:12345-192.168.202.128@tcp:104: Request processed in 134us (332us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764659.690328:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 169 00000100:00100000:1.0:1775764659.697633:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119022592 00000100:00100000:1.0:1775764659.697644:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119022592 00000100:00100000:1.0:1775764659.697702:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 170 00000100:00100000:1.0:1775764659.697711:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daf100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:44302:x1862022119022592:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764659.697727:0:46140:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b7a3 lrc: 5/1,0 mode: --/PR res: [0x240000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03050a8a expref: -99 pid: 58819 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.697740:0:46140:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b7a3 lrc: 5/1,0 mode: --/PR res: [0x240000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03050a8a expref: -99 pid: 58819 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.697750:0:46140:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b7a3 lrc: 5/1,0 mode: --/PR res: [0x240000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e03050a8a expref: -99 pid: 58819 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.697772:0:46140:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b7a3 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10080000020000 nid: local remote: 0x57b8ff0e03050a8a expref: -99 pid: 58819 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.697782:0:46140:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b7a3 lrc: 6/1,0 mode: PR/PR res: [0x240000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10080000020000 nid: local remote: 0x57b8ff0e03050a8a expref: -99 pid: 58819 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.697792:0:46140:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000009e189fb3/0x73cfd86cfc17b7a3 lrc: 6/1,0 mode: PR/PR res: [0x240000400:0x114:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x80000020000 nid: local remote: 0x57b8ff0e03050a8a expref: -99 pid: 58819 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.697821:0:46140:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b23b9c5fe00) 00000100:00100000:1.0:1775764659.697876:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daf100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:44302:x1862022119022592:12345-192.168.202.128@tcp:105: Request processed in 166us (345us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764659.697884:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 170 00000100:00100000:1.0:1775764659.707245:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200008:1.0:1775764659.980688:0:58836:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764659.980702:0:58836:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764659.980731:0:58836:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764659.980757:0:58836:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764659.980776:0:58836:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764659.980785:0:58836:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764659.980796:0:58836:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764659.980800:0:58836:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b787 00000080:00010000:1.0:1775764659.980810:0:58836:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b787 from it ffffa60b01e63cb8 00010000:00010000:1.0:1775764659.980815:0:58836:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764659.980825:0:58836:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764659.980854:0:58836:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764659.980860:0:58836:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764659.980866:0:58836:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764659.980875:0:58836:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764659.980881:0:58836:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764659.980884:0:58836:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764659.980887:0:58836:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b787 00000080:00010000:1.0:1775764659.980890:0:58836:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b787 from it ffffa60b01e63b58 00010000:00010000:1.0:1775764659.980893:0:58836:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764659.980901:0:58836:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1775764659.980909:0:58836:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764659.980915:0:58836:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21h.sanity-pcc, flags=8 00000080:00200008:1.0:1775764659.980922:0:58836:0:(file.c:6226:ll_getattr_dentry()) START file f21h.sanity-pcc:[0x200003ab1:0x35:0x0](ffff8b2390ec42c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764659.980925:0:58836:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab1:0x35:0x0](ffff8b2390ec42c8),name=f21h.sanity-pcc 00000002:00010000:1.0:1775764659.980936:0:58836:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x35:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764659.980943:0:58836:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044600/0x73cfd86cfc17b795 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050a6e expref: -99 pid: 58819 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764659.980951:0:58836:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044600/0x73cfd86cfc17b795 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050a6e expref: -99 pid: 58819 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764659.980956:0:58836:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764659.980959:0:58836:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x35:0x0](ffff8b2390ec42c8) 00000080:00010000:1.0:1775764659.980962:0:58836:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x35:0x0](ffff8b2390ec42c8) for lock 0x73cfd86cfc17b795 00000080:00010000:1.0:1775764659.980964:0:58836:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b795 from it ffffa60b01e63cb8 00010000:00010000:1.0:1775764659.980967:0:58836:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044600/0x73cfd86cfc17b795 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03050a6e expref: -99 pid: 58819 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764659.980973:0:58836:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044600/0x73cfd86cfc17b795 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050a6e expref: -99 pid: 58819 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764659.981006:0:58836:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764659.981012:0:58836:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764659.981018:0:58836:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764659.981026:0:58836:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1775764659.981031:0:58836:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764659.981035:0:58836:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764659.981037:0:58836:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b787 00000080:00010000:1.0:1775764659.981040:0:58836:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b787 from it ffffa60b01e63bf8 00010000:00010000:1.0:1775764659.981043:0:58836:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764659.981049:0:58836:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764659.981057:0:58836:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764659.981061:0:58836:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21h.sanity-pcc, flags=265 00000080:00200000:1.0:1775764659.981064:0:58836:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab1:0x35:0x0](ffff8b2390ec42c8), inode mode 81a4 mask 46 00000080:00200008:1.0:1775764659.981070:0:58836:0:(file.c:1057:ll_file_open()) START file f21h.sanity-pcc:[0x200003ab1:0x35:0x0](ffff8b2390ec42c8), flags 154402 00000002:00010000:1.0:1775764659.981083:0:58836:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x35:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240114403 00010000:00010000:1.0:1775764659.981134:0:58836:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741200/0x73cfd86cfc17b7b1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 58836 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.981142:0:58836:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741200/0x73cfd86cfc17b7b1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 58836 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764659.981155:0:58836:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764659.981167:0:58836:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ff100 x1862022098495872/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764659.981177:0:58836:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741200/0x73cfd86cfc17b7b1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 58836 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764659.981697:0:58836:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:58836:1862022098495872:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764659.981980:0:58836:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b8759240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764659.987681:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119022976 00000100:00100000:1.0:1775764659.987692:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119022976 00000100:00100000:1.0:1775764659.987783:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 172 00000100:00100000:1.0:1775764659.987793:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daf100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42324:x1862022119022976:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764659.987811:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044600/0x73cfd86cfc17b795 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x35:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03050a6e expref: -99 pid: 58819 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764659.987890:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764659.987897:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daf100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42324:x1862022119022976:12345-192.168.202.128@tcp:104: Request processed in 107us (536us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764659.987904:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 172 00000100:00100000:1.0:1775764659.995948:0:58836:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:58836:1862022098495872:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764659.995981:0:58836:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CW ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741200/0x73cfd86cfc17b7b1 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ab4 expref: -99 pid: 58836 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.995998:0:58836:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741200/0x73cfd86cfc17b7b1 lrc: 4/1,0 mode: --/CW res: [0x200003ab1:0x35:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ab4 expref: -99 pid: 58836 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.996011:0:58836:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741200/0x73cfd86cfc17b7b1 lrc: 4/1,0 mode: --/CW res: [0x200003ab1:0x35:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ab4 expref: -99 pid: 58836 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.996026:0:58836:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741200/0x73cfd86cfc17b7b1 lrc: 4/1,0 mode: CW/CW res: [0x200003ab1:0x35:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050ab4 expref: -99 pid: 58836 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.996040:0:58836:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741200/0x73cfd86cfc17b7b1 lrc: 4/1,0 mode: CW/CW res: [0x200003ab1:0x35:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ab4 expref: -99 pid: 58836 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.996057:0:58836:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741200/0x73cfd86cfc17b7b1 lrc: 5/1,1 mode: CW/CW res: [0x200003ab1:0x35:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050ab4 expref: -99 pid: 58836 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.996066:0:58836:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741200/0x73cfd86cfc17b7b1 lrc: 5/1,1 mode: CW/CW res: [0x200003ab1:0x35:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050ab4 expref: -99 pid: 58836 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.996075:0:58836:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741200/0x73cfd86cfc17b7b1 lrc: 4/0,1 mode: CW/CW res: [0x200003ab1:0x35:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050ab4 expref: -99 pid: 58836 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764659.996099:0:58836:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23864ff100 x1862022098495872/t34359738922(34359738922) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 1056/1144 e 0 to 0 dl 1775764675 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764659.996111:0:58836:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864ff100 x1862022098495872/t34359738922(34359738922) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 1056/1144 e 0 to 0 dl 1775764675 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764659.996126:0:58836:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741200/0x73cfd86cfc17b7b1 lrc: 3/0,1 mode: CW/CW res: [0x200003ab1:0x35:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ab4 expref: -99 pid: 58836 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.996138:0:58836:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 8 res 8589949617/53 (0 0) 00000080:00200000:1.0:1775764659.996145:0:58836:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764659.996243:0:58836:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab1:0x35:0x0], updating i_size 0 00000080:00010000:1.0:1775764659.996248:0:58836:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x35:0x0](ffff8b2390ec42c8) for lock 0x73cfd86cfc17b7b1 00000080:00010000:1.0:1775764659.996255:0:58836:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b7b1 from it ffffa60b01e63bf0 00010000:00010000:1.0:1775764659.996258:0:58836:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741200/0x73cfd86cfc17b7b1 lrc: 3/0,1 mode: CW/CW res: [0x200003ab1:0x35:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050ab4 expref: -99 pid: 58836 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764659.996267:0:58836:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741200/0x73cfd86cfc17b7b1 lrc: 2/0,0 mode: CW/CW res: [0x200003ab1:0x35:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ab4 expref: -99 pid: 58836 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764659.996292:0:58836:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864ff100 x1862022098495872/t34359738922(34359738922) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 1056/1144 e 0 to 0 dl 1775764675 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764659.996395:0:58836:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab1:0x35:0x0](ffff8b2390ec42c8) cmd=c00466fc arg=7ffe1e87871c 00000080:00200008:1.0:1775764659.996421:0:58836:0:(file.c:447:ll_file_release()) START file f21h.sanity-pcc:[0x200003ab1:0x35:0x0](ffff8b2390ec42c8), flags 154002 00010000:00010000:1.0:1775764659.996439:0:58836:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741200/0x73cfd86cfc17b7b1 lrc: 2/0,0 mode: CW/CW res: [0x200003ab1:0x35:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ab4 expref: -99 pid: 58836 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:1.0:1775764660.806973:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098496256:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764660.807000:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098496512:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764660.807014:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098496768:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764660.807046:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098496384:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764660.807059:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098496640:192.168.202.128@tcp:400:kworker.0 00010000:00010000:1.0:1775764661.054947:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764663.107538:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764665.145188:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764665.911839:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098497152:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764665.912291:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098497408:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764665.912335:0:2377:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:1.0:1775764665.912342:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098497024:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764665.912369:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098497280:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764665.912390:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098497536:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764665.916120:0:2377:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b2384b87800: committing for last_committed 34359738421 gen 2 00000100:00100000:1.0:1775764665.916172:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098497280:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764665.916187:0:2377:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b2384b87800: committing for last_committed 34359738429 gen 1 00000100:00100000:1.0:1775764665.916202:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098497408:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764665.923048:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098497024:192.168.202.128@tcp:400:kworker.0 00010000:00010000:1.0:1775764668.215786:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764670.263683:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1775764676.263708:0:59545:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764676.263723:0:59545:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764676.263743:0:59545:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764676.263760:0:59545:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:1.0:1775764676.263775:0:59545:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:1.0:1775764676.263786:0:59545:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764676.263794:0:59545:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764676.263797:0:59545:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b787 00000080:00010000:1.0:1775764676.263803:0:59545:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b787 from it ffffa60b01e03cb8 00010000:00010000:1.0:1775764676.263807:0:59545:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:1.0:1775764676.263815:0:59545:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:1.0:1775764676.268469:0:59546:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764676.268495:0:59546:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764676.268513:0:59546:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:1.0:1775764676.268527:0:59546:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:1.0:1775764676.268535:0:59546:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764676.268542:0:59546:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764676.268545:0:59546:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b787 00000080:00010000:1.0:1775764676.268551:0:59546:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b787 from it ffffa60b01e2bbb0 00010000:00010000:1.0:1775764676.268554:0:59546:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:1.0:1775764676.268562:0:59546:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:1.0:1775764676.268577:0:59546:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764676.268584:0:59546:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 100000 00000080:00200000:1.0:1775764676.268604:0:59546:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764676.268610:0:59546:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764676.268617:0:59546:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764676.268624:0:59546:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:1.0:1775764676.268630:0:59546:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764676.268634:0:59546:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764676.268636:0:59546:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b787 00000080:00010000:1.0:1775764676.268639:0:59546:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b787 from it ffffa60b01e2bbf8 00010000:00010000:1.0:1775764676.268642:0:59546:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764676.268649:0:59546:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:1.0:1775764676.268656:0:59546:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764676.268662:0:59546:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=.lustre, flags=17 00000080:00200000:1.0:1775764676.268667:0:59546:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000002:0x1:0x0](ffff8b23899080c8), inode mode 41ed mask 1 00000080:00200000:1.0:1775764676.268679:0:59546:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=fid, dir=[0x200000002:0x1:0x0](ffff8b23899080c8), file ffff8b238b5be300, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1775764676.268684:0:59546:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=fid, dir=[0x200000002:0x1:0x0](ffff8b23899080c8), intent=open 00000002:00010000:1.0:1775764676.268697:0:59546:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: fid,[0x0:0x0:0x0]) in obj [0x200000002:0x1:0x0], intent: open flags 0100001 00010000:00010000:1.0:1775764676.268747:0:59546:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741c00/0x73cfd86cfc17b7bf lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 59546 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764676.268754:0:59546:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741c00/0x73cfd86cfc17b7bf lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 59546 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764676.268765:0:59546:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764676.268777:0:59546:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fce00 x1862022098499584/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764676.268787:0:59546:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741c00/0x73cfd86cfc17b7bf lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 59546 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764676.268807:0:59546:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:59546:1862022098499584:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764676.268979:0:59546:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e10c0 going to sleep for 11 seconds 00010000:00010000:1.0:1775764676.286596:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764676.286605:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741c00/0x73cfd86cfc17b7bf lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03050afa expref: -99 pid: 59546 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764676.286615:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8741c00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764676.286620:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741c00/0x73cfd86cfc17b7bf lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03050afa expref: -99 pid: 59546 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764676.286627:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741c00/0x73cfd86cfc17b7bf lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03050afa expref: -99 pid: 59546 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764676.286635:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741c00/0x73cfd86cfc17b7bf lrc: 3/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03050afa expref: -99 pid: 59546 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764676.286656:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8606000/0x73cfd86cfc17b0cd lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0304f273 expref: -99 pid: 51094 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1775764676.286694:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741c00/0x73cfd86cfc17b7bf lrc: 2/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03050afa expref: -99 pid: 59546 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764676.286700:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764676.286728:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741c00/0x73cfd86cfc17b7bf lrc: 1/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03050afa expref: -99 pid: 59546 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764676.286736:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741c00/0x73cfd86cfc17b7bf lrc: 0/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03050afa expref: -99 pid: 59546 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764676.286748:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764676.286750:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764676.286751:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764676.286759:0:2376:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764676.286778:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098499840:192.168.202.128@tcp:103:ldlm_bl.0 00000080:00200000:1.0:1775764676.516087:0:59554:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764676.516118:0:59554:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764676.516138:0:59554:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764676.516155:0:59554:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:1.0:1775764676.516163:0:59554:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764676.516171:0:59554:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764676.516175:0:59554:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b787 00000080:00010000:1.0:1775764676.516182:0:59554:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b787 from it ffffa60b01e5bbb0 00010000:00010000:1.0:1775764676.516187:0:59554:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764676.516195:0:59554:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea400/0x73cfd86cfc17b787 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030509fe expref: -99 pid: 58802 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:1.0:1775764676.516212:0:59554:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764676.516221:0:59554:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 504000 00000080:00200080:1.0:1775764676.516249:0:59554:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=8080667f arg=7ffc796d8fb0 00000080:00200008:1.0:1775764676.516263:0:59554:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 504000 00000100:00100000:1.0:1775764676.656298:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764676.656913:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 173 00000100:00100000:1.0:1775764676.656925:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23864fd500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:44493:x1862022119026688:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764676.656961:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb000/0x73cfd86cfc17b7b8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03050ade expref: -99 pid: 59286 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764676.657099:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764676.657106:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23864fd500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:44493:x1862022119026688:12345-192.168.202.128@tcp:104: Request processed in 184us (821us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764676.657114:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 173 00000080:00200000:1.0:1775764676.676589:0:59560:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x37:0x0] updating i_size 0 i_blocks 1 00000020:00010000:1.0:1775764676.676595:0:59560:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2389ea8ff8 (ffff8b238a3ffab8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764676.676598:0:59560:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389ea8ff8 (ffff8b238a3ffab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764676.676603:0:59560:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23b8563e30 (ffff8b238a3ff7e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764676.676608:0:59560:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389ea8ff8 (ffff8b238a3ffab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764676.676611:0:59560:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23b8563e30 (ffff8b238a3ffab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764676.676651:0:59560:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x37:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1775764676.676748:0:59560:0:(file.c:2612:do_file_write_iter()) START file f21i.sanity-pcc:[0x200003ab1:0x37:0x0], ppos: 0, count: 20 00000080:00200000:1.0:1775764676.676767:0:59560:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 20 bytes 00000080:00200000:1.0:1775764676.676773:0:59560:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 20 00000080:00200000:1.0:1775764676.676802:0:59560:0:(file.c:2058:ll_file_io_generic()) f21i.sanity-pcc: write ppos: 0, bytes: 20 00000080:00200000:1.0:1775764676.676807:0:59560:0:(file.c:1815:ll_io_set_mirror()) f21i.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764676.676813:0:59560:0:(cl_io.c:214:cl_io_rw_init()) header@0000000017fde005[0x110, 1, [0x200003ab1:0x37:0x0] hash] 00000020:00200000:1.0:1775764676.676815:0:59560:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 20) 0 0 00000080:00200000:1.0:1775764676.676819:0:59560:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x37:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764676.676831:0:59560:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1775764676.676839:0:59560:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 20) 20, 20 00020000:00200000:1.0:1775764676.676842:0:59560:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764676.676869:0:59560:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1775764676.676881:0:59560:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764676.676886:0:59560:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a3180b0, osc lock ffff8b2382cf7c80, flags 0x0 00010000:00010000:1.0:1775764676.676901:0:59560:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b23831ed400 type 11 mode 2 res 9663677440/277 (0 4095) 00010000:00010000:1.0:1775764676.677235:0:59560:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000da82aaac/0x73cfd86cfc17b7d4 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x115:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 59560 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764676.677247:0:59560:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000da82aaac/0x73cfd86cfc17b7d4 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x115:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 59560 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764676.677261:0:59560:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000da82aaac/0x73cfd86cfc17b7d4 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x115:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 59560 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764676.677289:0:59560:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:59560:1862022098500480:192.168.202.128@tcp:101:bash.0 00000100:00100000:1.0:1775764676.677383:0:59560:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e19c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764676.683083:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119026816 00000100:00100000:1.0:1775764676.683097:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119026816 00000100:00100000:1.0:1775764676.683172:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 174 00000100:00100000:1.0:1775764676.683184:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dad180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42946:x1862022119026816:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764676.683214:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 000000003c855c97/0x73cfd86cfc17b7cd lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x115:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e03050b2b expref: -99 pid: 59560 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764676.683313:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764676.683320:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dad180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42946:x1862022119026816:12345-192.168.202.128@tcp:104: Request processed in 139us (426us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764676.683328:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 174 00000100:00100000:1.0:1775764676.690001:0:59560:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:59560:1862022098500480:192.168.202.128@tcp:101:bash.0 00010000:00010000:1.0:1775764676.690031:0:59560:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000da82aaac/0x73cfd86cfc17b7d4 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03050b32 expref: -99 pid: 59560 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764676.690196:0:59560:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000da82aaac/0x73cfd86cfc17b7d4 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e03050b32 expref: -99 pid: 59560 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1775764676.690213:0:59560:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000da82aaac/0x73cfd86cfc17b7d4 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000020000 nid: local remote: 0x57b8ff0e03050b32 expref: -99 pid: 59560 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764676.690228:0:59560:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000da82aaac/0x73cfd86cfc17b7d4 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e03050b32 expref: -99 pid: 59560 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764676.690237:0:59560:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000da82aaac/0x73cfd86cfc17b7d4 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e03050b32 expref: -99 pid: 59560 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1775764676.690279:0:59560:0:(vvp_io.c:1286:vvp_io_write_start()) f21i.sanity-pcc: write [0, 20) append 0 append lockpos 0 00000080:00200000:1.0:1775764676.690305:0:59560:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 20 bytes 00000080:00200000:1.0:1775764676.690381:0:59560:0:(rw26.c:858:ll_write_end()) pos 0, len 20, copied 20 00000080:00200000:1.0:1775764676.690403:0:59560:0:(rw26.c:916:ll_write_end()) page@000000000b4d8e38[3 000000003f414d76 2 1 000000007cfbe91e] 00000080:00200000:1.0:1775764676.690407:0:59560:0:(rw26.c:916:ll_write_end()) vmpage @00000000b2a53741 17ffffc0001001 3:0 ffff8b238a05fd20 0 lru 00000080:00200000:1.0:1775764676.690414:0:59560:0:(rw26.c:916:ll_write_end()) osc-page@00000000a24116f5 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 00000000bf12bc0b 00000000cb6fe058 > 3< 0 0 > 4< 0 0 8 18743296 - | - - + - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764676.690417:0:59560:0:(rw26.c:916:ll_write_end()) end page@000000000b4d8e38 00000080:00200000:1.0:1775764676.690418:0:59560:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764676.690423:0:59560:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 20 prio 0 00000008:00010000:1.0:1775764676.690447:0:59560:0:(osc_cache.c:848:osc_extent_find()) ### extent: 00000000ef5674d2 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000da82aaac/0x73cfd86cfc17b7d4 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03050b32 expref: -99 pid: 59560 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1775764676.690491:0:59560:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b238990ccb8, count 1, dirtied 1 00000080:00200000:1.0:1775764676.690497:0:59560:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 20 bytes, tot: 20 00000080:00200000:1.0:1775764676.690500:0:59560:0:(vvp_io.c:1353:vvp_io_write_start()) f21i.sanity-pcc: write bytes 0, result: 20 00000020:00010000:1.0:1775764676.690522:0:59560:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb7b78 (ffff8b2387cba758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764676.690526:0:59560:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb7b78 (ffff8b2387cba758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764676.690529:0:59560:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a3180b0 (ffff8b2387cba5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764676.690533:0:59560:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000da82aaac/0x73cfd86cfc17b7d4 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e03050b32 expref: -99 pid: 59560 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764676.690542:0:59560:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000da82aaac/0x73cfd86cfc17b7d4 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x115:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03050b32 expref: -99 pid: 59560 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:1.0:1775764676.690558:0:59560:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb7b78 (ffff8b2387cba758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764676.690562:0:59560:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a3180b0 (ffff8b2387cba758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764676.690571:0:59560:0:(vvp_io.c:517:vvp_io_advance()) advancing 20 bytes 00000080:00200000:1.0:1775764676.690578:0:59560:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:1.0:1775764676.690606:0:59560:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x37:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764676.690610:0:59560:0:(file.c:2206:ll_file_io_generic()) f21i.sanity-pcc: 2 io complete with rc: 0, result: 20, restart: 0 00000080:00200000:1.0:1775764676.690618:0:59560:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x37:0x0] updating i_size 20 i_blocks 1 00000080:00200000:1.0:1775764676.690623:0:59560:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 20 00000080:00200008:1.0:1775764676.690673:0:59560:0:(file.c:447:ll_file_release()) START file f21i.sanity-pcc:[0x200003ab1:0x37:0x0](ffff8b238990cb08), flags 100001 00000002:00100000:1.0:1775764676.690731:0:59560:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864f2300 x1862022098500096/t34359738925(34359738925) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764692 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764676.690743:0:59560:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764676.690748:0:59560:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fdf80 x1862022098500864/t0(0) o35->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764676.690772:0:59560:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:59560:1862022098500864:192.168.202.128@tcp:35:bash.0 00000100:00100000:1.0:1775764676.690856:0:59560:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e19c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764676.696026:0:59560:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job bash:7e6eda44-5508-487e-b17f-ff28ad9d3512:59560:1862022098500864:192.168.202.128@tcp:35:bash.0 00000002:00100000:1.0:1775764676.696085:0:59560:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864f2300 x1862022098500096/t34359738925(34359738925) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764692 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764676.803336:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098501632:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764676.822792:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098501632:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764676.824883:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098502016:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764676.832088:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098502016:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764676.842226:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119027200 00000100:00100000:1.0:1775764676.842239:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119027200 00000100:00100000:1.0:1775764676.842310:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 177 00000100:00100000:1.0:1775764676.842321:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dadc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42328:x1862022119027200:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764676.842345:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b9c5e200/0x73cfd86cfc17b7c6 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03050b1d expref: -99 pid: 59560 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764676.842497:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764676.842505:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dadc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42328:x1862022119027200:12345-192.168.202.128@tcp:104: Request processed in 186us (577us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764676.842513:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 177 00000100:00100000:1.0:1775764677.008232:0:59576:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59576:1862022098502912:192.168.202.128@tcp:34:lfs.0 00000080:00200008:1.0:1775764677.008342:0:59576:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 304000 00000100:00100000:1.0:1775764677.219523:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098504192:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764677.428886:0:59603:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59603:1862022098504448:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764677.428941:0:59603:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044200/0x73cfd86cfc17b805 lrc: 4/1,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050be1 expref: -99 pid: 59603 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764677.428956:0:59603:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044200/0x73cfd86cfc17b805 lrc: 4/1,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050be1 expref: -99 pid: 59603 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764677.428969:0:59603:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044200/0x73cfd86cfc17b805 lrc: 4/1,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050be1 expref: -99 pid: 59603 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764677.428982:0:59603:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044200/0x73cfd86cfc17b805 lrc: 4/1,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050be1 expref: -99 pid: 59603 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764677.428998:0:59603:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b2382daca80 x1862022098504448/t34359738943(34359738943) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764693 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764677.429011:0:59603:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382daca80 x1862022098504448/t34359738943(34359738943) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764693 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764677.429025:0:59603:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044200/0x73cfd86cfc17b805 lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050be1 expref: -99 pid: 59603 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764677.429057:0:59603:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044200/0x73cfd86cfc17b805 lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050be1 expref: -99 pid: 59603 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764677.429067:0:59603:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 16 res 8589949617/55 (0 0) 00000080:00200000:1.0:1775764677.429074:0:59603:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764677.429279:0:59603:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab1:0x37:0x0], updating i_size 20 00000080:00200000:1.0:1775764677.429300:0:59603:0:(file.c:6766:ll_layout_conf()) [0x200003ab1:0x37:0x0]: layout version change: 1 -> 1 00000080:00010000:1.0:1775764677.429314:0:59603:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x37:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17b805 00000080:00010000:1.0:1775764677.429322:0:59603:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b805 from it ffffa60b01ebbbf0 00010000:00010000:1.0:1775764677.429326:0:59603:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044200/0x73cfd86cfc17b805 lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03050be1 expref: -99 pid: 59603 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764677.429337:0:59603:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044200/0x73cfd86cfc17b805 lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050be1 expref: -99 pid: 59603 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764677.429369:0:59603:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382daca80 x1862022098504448/t34359738943(34359738943) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764693 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764677.429935:0:59603:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab1:0x37:0x0](ffff8b238990cb08) cmd=802066d3 arg=7fff84120080 00800000:00000080:1.0:1775764677.429955:0:59603:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=802066d3 len=384 karg=ffff8b2383ebda00 uarg=0000000000000000 00000002:00000080:1.0:1775764677.429962:0:59603:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2384b87800: cmd=802066d3 len=384 karg=ffff8b2383ebda00 uarg=0000000000000000 00000100:00100000:1.0:1775764677.430009:0:59603:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59603:1862022098504576:192.168.202.128@tcp:54:lfs.0 00000100:00100000:1.0:1775764677.430119:0:59603:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e1540 going to sleep for 11 seconds 00000100:00100000:1.0:1775764677.437029:0:59603:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59603:1862022098504576:192.168.202.128@tcp:54:lfs.0 00000080:00200008:1.0:1775764677.437115:0:59603:0:(file.c:447:ll_file_release()) START file f21i.sanity-pcc:[0x200003ab1:0x37:0x0](ffff8b238990cb08), flags 104000 00010000:00010000:1.0:1775764677.437160:0:59603:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044200/0x73cfd86cfc17b805 lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050be1 expref: -99 pid: 59603 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:1.0:1775764677.570613:0:59613:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764677.570631:0:59613:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764677.570659:0:59613:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764677.570681:0:59613:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764677.570698:0:59613:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764677.570707:0:59613:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764677.570716:0:59613:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764677.570721:0:59613:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b7db 00000080:00010000:1.0:1775764677.570731:0:59613:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b7db from it ffffa60b01efbcb8 00010000:00010000:1.0:1775764677.570735:0:59613:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764677.570745:0:59613:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764677.570777:0:59613:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764677.570784:0:59613:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764677.570790:0:59613:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764677.570799:0:59613:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764677.570807:0:59613:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764677.570811:0:59613:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764677.570814:0:59613:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b7db 00000080:00010000:1.0:1775764677.570817:0:59613:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b7db from it ffffa60b01efbb58 00010000:00010000:1.0:1775764677.570820:0:59613:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764677.570828:0:59613:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00000080:00200000:1.0:1775764677.570837:0:59613:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764677.570845:0:59613:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21i.sanity-pcc, flags=8 00000080:00200008:1.0:1775764677.570852:0:59613:0:(file.c:6226:ll_getattr_dentry()) START file f21i.sanity-pcc:[0x200003ab1:0x37:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764677.570856:0:59613:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab1:0x37:0x0](ffff8b238990cb08),name=f21i.sanity-pcc 00000002:00010000:1.0:1775764677.570868:0:59613:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x37:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764677.570876:0:59613:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ebe00/0x73cfd86cfc17b7f7 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050bb0 expref: -99 pid: 59579 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764677.570884:0:59613:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ebe00/0x73cfd86cfc17b7f7 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050bb0 expref: -99 pid: 59579 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764677.570893:0:59613:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764677.570897:0:59613:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x37:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764677.570900:0:59613:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x37:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17b7f7 00000080:00010000:1.0:1775764677.570904:0:59613:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b7f7 from it ffffa60b01efbcb8 00010000:00010000:1.0:1775764677.570908:0:59613:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ebe00/0x73cfd86cfc17b7f7 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03050bb0 expref: -99 pid: 59579 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764677.570917:0:59613:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ebe00/0x73cfd86cfc17b7f7 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050bb0 expref: -99 pid: 59579 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764677.570947:0:59613:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764677.570953:0:59613:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764677.570960:0:59613:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764677.570969:0:59613:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00000080:00200000:1.0:1775764677.570976:0:59613:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764677.570980:0:59613:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764677.570983:0:59613:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b7db 00000080:00010000:1.0:1775764677.570987:0:59613:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b7db from it ffffa60b01efbbf8 00010000:00010000:1.0:1775764677.570990:0:59613:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764677.570998:0:59613:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764677.571006:0:59613:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764677.571011:0:59613:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21i.sanity-pcc, flags=265 00000080:00200000:1.0:1775764677.571015:0:59613:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab1:0x37:0x0](ffff8b238990cb08), inode mode 81a4 mask 44 00000080:00200008:1.0:1775764677.571023:0:59613:0:(file.c:1057:ll_file_open()) START file f21i.sanity-pcc:[0x200003ab1:0x37:0x0](ffff8b238990cb08), flags 100000 00000080:00200080:1.0:1775764677.571240:0:59613:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab1:0x37:0x0](ffff8b238990cb08) cmd=400866fc arg=7ffe737e5218 00800000:00000080:1.0:1775764677.571321:0:59613:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=401866d9 len=56 karg=ffff8b23891edc00 uarg=0000000000000000 00000002:00000080:1.0:1775764677.571330:0:59613:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2384b87800: cmd=401866d9 len=56 karg=ffff8b23891edc00 uarg=0000000000000000 00010000:00010000:1.0:1775764677.571363:0:59613:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ebe00/0x73cfd86cfc17b7f7 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03050bb0 expref: -99 pid: 59579 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764677.571617:0:59613:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab1:0x37:0x0]: losing layout lock 00010000:00010000:1.0:1775764677.571647:0:59613:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044200/0x73cfd86cfc17b805 lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03050be1 expref: -99 pid: 59603 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764677.571666:0:59613:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab1:0x37:0x0]: losing layout lock 00010000:00010000:1.0:1775764677.571679:0:59613:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb800/0x73cfd86cfc17b7f0 lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03050b7f expref: -99 pid: 59566 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764677.571694:0:59613:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab1:0x37:0x0]: losing layout lock 00010000:00010000:1.0:1775764677.571721:0:59613:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ebe00/0x73cfd86cfc17b7f7 lrc: 1/0,0 mode: --/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03050bb0 expref: -99 pid: 59579 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764677.571729:0:59613:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044200/0x73cfd86cfc17b805 lrc: 1/0,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03050be1 expref: -99 pid: 59603 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764677.571736:0:59613:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb800/0x73cfd86cfc17b7f0 lrc: 1/0,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03050b7f expref: -99 pid: 59566 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764677.571742:0:59613:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00000100:00100000:1.0:1775764677.571773:0:59613:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59613:1862022098504832:192.168.202.128@tcp:103:lfs.0 00000100:00100000:1.0:1775764677.571949:0:59613:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e1c00 going to sleep for 11 seconds 00000100:00100000:1.0:1775764677.576918:0:59613:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59613:1862022098504832:192.168.202.128@tcp:103:lfs.0 00010000:00010000:1.0:1775764677.576970:0:59613:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ebe00/0x73cfd86cfc17b7f7 lrc: 0/0,0 mode: --/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03050bb0 expref: -99 pid: 59579 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764677.576992:0:59613:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044200/0x73cfd86cfc17b805 lrc: 0/0,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03050be1 expref: -99 pid: 59603 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764677.577004:0:59613:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb800/0x73cfd86cfc17b7f0 lrc: 0/0,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03050b7f expref: -99 pid: 59566 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1775764677.577032:0:59613:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 58, max 7 00000100:00100000:1.0:1775764677.577052:0:59613:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fe680 x1862022098504960/t0(0) o58->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764677.577080:0:59613:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59613:1862022098504960:192.168.202.128@tcp:58:lfs.0 00000080:00200000:1.0:1775764677.687777:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x32:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764677.687793:0:8:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab1:0x32:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 3 00020000:00200000:1.0:1775764677.687797:0:8:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1775764677.687816:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:1.0:1775764677.687870:0:8:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 00000000f6451404 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00000008:00010000:1.0:1775764677.687899:0:8:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000f6451404 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764677.687965:0:8:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00000008:00100000:1.0:1775764677.688079:0:8:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23864fd880 - object 0x240000400:272 offset 0<>29 00000080:00200000:1.0:1775764677.688170:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x32:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:1.0:1775764677.688221:0:2377:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764677.688243:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098505216:192.168.202.128@tcp:4:bash.0 00000008:00010000:1.0:1775764677.694341:0:2377:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 00000000f6451404 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00000008:00010000:1.0:1775764677.694415:0:2377:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f6451404 ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000b3d032fc/0x73cfd86cfc17b668 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x110:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x57b8ff0e03050685 expref: -99 pid: 58062 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00000100:00100000:1.0:1775764677.694498:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098505216:192.168.202.128@tcp:4:bash.0 00000100:00100000:1.0:1775764678.033411:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764678.053444:0:59614:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:59614:1862022098505344:192.168.202.128@tcp:53:lhsmtool_posix.0 00000002:00000080:1.0:1775764678.053525:0:59614:0:(mdc_request.c:1758:mdc_ioc_fid2path()) path got [0x200000007:0x1:0x0] from 18446744073709551615 #0: f21i.sanity-pcc 00000080:00200080:1.0:1775764678.053730:0:59614:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=c01066f8 arg=21659e8 00800000:00000080:1.0:1775764678.053742:0:59614:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=c01066f8 len=16 karg=ffffa60b01eb3cf0 uarg=ffffa60b01eb3cc0 00000080:00200080:1.0:1775764678.054355:0:59614:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000002:0x2:0x0](ffff8b238990e3c8) cmd=c1406916 arg=7f38f0000ce0 00000100:00100000:1.0:1775764678.054437:0:59614:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:59614:1862022098505472:192.168.202.128@tcp:34:lhsmtool_posix.0 00000100:00100000:1.0:1775764678.056199:0:59614:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e16c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764678.070934:0:59614:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:59614:1862022098505472:192.168.202.128@tcp:34:lhsmtool_posix.0 00000080:00200000:1.0:1775764678.071076:0:59614:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000002:0x2:0x0](ffff8b238990e3c8), inode mode 4040 mask 1 00000080:00200000:1.0:1775764678.071102:0:59614:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=0x200003ab1:0x37:0x0, dir=[0x200000002:0x2:0x0](ffff8b238990e3c8), file ffff8b239847e780, open_flags 68800, mode 0 opened 0 00000080:00200000:1.0:1775764678.071112:0:59614:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=0x200003ab1:0x37:0x0, dir=[0x200000002:0x2:0x0](ffff8b238990e3c8), intent=open 00000002:00010000:1.0:1775764678.071149:0:59614:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x37:0x0]) in obj [0x200000002:0x2:0x0], intent: open flags 041504001 00010000:00010000:1.0:1775764678.071246:0:59614:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b813 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 59614 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.071257:0:59614:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b813 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 59614 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764678.071270:0:59614:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764678.071280:0:59614:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fdc00 x1862022098505728/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764678.071292:0:59614:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b813 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 59614 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764678.071320:0:59614:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:59614:1862022098505728:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764678.071857:0:59614:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e16c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764678.082599:0:59614:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:59614:1862022098505728:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:1.0:1775764678.082653:0:59614:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b813 lrc: 4/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 59614 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.082670:0:59614:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b813 lrc: 4/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 59614 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.082678:0:59614:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b813 lrc: 4/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 59614 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.082685:0:59614:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b813 lrc: 3/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 59614 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.082695:0:59614:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b813 lrc: 4/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 59614 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.082706:0:59614:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8741600 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764678.082710:0:59614:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b813 lrc: 4/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 59614 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.082717:0:59614:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b813 lrc: 4/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 59614 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.082724:0:59614:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b813 lrc: 5/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 59614 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.082731:0:59614:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764678.082738:0:59614:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b813 lrc: 3/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 59614 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.082744:0:59614:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b813 lrc: 0/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 59614 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764678.082765:0:59614:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23864fdc00 x1862022098505728/t34359738944(34359738944) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764694 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764678.082778:0:59614:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fdc00 x1862022098505728/t34359738944(34359738944) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764694 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764678.082796:0:59614:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000002:0x2:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764678.082895:0:59614:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab1:0x37:0x0], updating i_size 20 00000080:00200000:1.0:1775764678.082900:0:59614:0:(namei.c:147:ll_iget()) got inode: [0x200003ab1:0x37:0x0](ffff8b238990cb08): rc = 0 00000080:00010000:1.0:1775764678.082905:0:59614:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x37:0x0](ffff8b238990cb08) for lock 0x0 00000080:00200008:1.0:1775764678.082946:0:59614:0:(file.c:1057:ll_file_open()) START file 0x200003ab1:0x37:0x0:[0x200003ab1:0x37:0x0](ffff8b238990cb08), flags 1504000 00000002:00100000:1.0:1775764678.082972:0:59614:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fdc00 x1862022098505728/t34359738944(34359738944) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764694 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764678.083022:0:59614:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fdc00 x1862022098505728/t34359738944(34359738944) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764694 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764678.083058:0:59614:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764678.083063:0:59614:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fe300 x1862022098505984/t0(0) o35->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764678.083093:0:59614:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:59614:1862022098505984:192.168.202.128@tcp:35:lhsmtool_posix.0 00000100:00100000:1.0:1775764678.083180:0:59614:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e16c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764678.096391:0:59614:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:59614:1862022098505984:192.168.202.128@tcp:35:lhsmtool_posix.0 00000002:00100000:1.0:1775764678.096439:0:59614:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fdc00 x1862022098505728/t34359738944(34359738944) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764694 ref 2 fl Complete:RQU/604/0 rc 301/301 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764678.096554:0:59614:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab1:0x37:0x0](ffff8b238990cb08), inode mode 81a4 mask 44 00000080:00200080:1.0:1775764678.096594:0:59614:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=c0086696 arg=7f38f0000ce0 00800000:00000080:1.0:1775764678.096616:0:59614:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=c0086696 len=4128 karg=ffff8b23ab52e000 uarg=0000000000000000 00000002:00000080:1.0:1775764678.096633:0:59614:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2384b87800: cmd=c0086696 len=4128 karg=ffff8b23ab52e000 uarg=0000000000000000 00000002:00000080:1.0:1775764678.096641:0:59614:0:(mdc_request.c:1738:mdc_ioc_fid2path()) path get [0x200003ab1:0x37:0x0] from 18446744073709551615 #0 00000100:00100000:1.0:1775764678.096679:0:59614:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:59614:1862022098506112:192.168.202.128@tcp:53:lhsmtool_posix.0 00000100:00100000:1.0:1775764678.096754:0:59614:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e16c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764678.117711:0:59614:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:59614:1862022098506112:192.168.202.128@tcp:53:lhsmtool_posix.0 00000002:00000080:1.0:1775764678.117753:0:59614:0:(mdc_request.c:1758:mdc_ioc_fid2path()) path got [0x200000007:0x1:0x0] from 18446744073709551615 #0: f21i.sanity-pcc 00000080:00200000:1.0:1775764678.117854:0:59614:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764678.117879:0:59614:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764678.117905:0:59614:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764678.117940:0:59614:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764678.117949:0:59614:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764678.117957:0:59614:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764678.117961:0:59614:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b7db 00000080:00010000:1.0:1775764678.117972:0:59614:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b7db from it ffffa60b01eb3bf8 00010000:00010000:1.0:1775764678.117975:0:59614:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764678.117984:0:59614:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1775764678.118006:0:59614:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764678.118044:0:59614:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764678.118051:0:59614:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764678.118057:0:59614:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764678.118066:0:59614:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1775764678.118073:0:59614:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764678.118077:0:59614:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764678.118080:0:59614:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b7db 00000080:00010000:1.0:1775764678.118084:0:59614:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b7db from it ffffa60b01eb3b38 00010000:00010000:1.0:1775764678.118088:0:59614:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764678.118095:0:59614:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:1.0:1775764678.118104:0:59614:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 3 00000080:00200000:1.0:1775764678.118110:0:59614:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=. :VOLATILE:0000:F8E8A0F:fd=09, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), file ffff8b239847fb00, open_flags 280c2, mode 8180 opened 0 00000080:00200000:1.0:1775764678.118118:0:59614:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=. :VOLATILE:0000:F8E8A0F:fd=09, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=open|creat 00000002:00010000:1.0:1775764678.118149:0:59614:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: . :VOLATILE:0000:F8E8A0F:fd=09,[0x200003ab1:0x39:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0500303 00010000:00010000:1.0:1775764678.118165:0:59614:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764678.118235:0:59614:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764678.118244:0:59614:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764678.118247:0:59614:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044000/0x73cfd86cfc17b7db lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03050b40 expref: -99 pid: 59566 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764678.118292:0:59614:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b81a lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 59614 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.118299:0:59614:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b81a lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 59614 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764678.118314:0:59614:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764678.118323:0:59614:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fdc00 x1862022098506368/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 1088/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764678.118333:0:59614:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b81a lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 59614 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764678.118357:0:59614:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:59614:1862022098506368:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764678.265974:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764678.280644:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 179 00000100:00100000:1.0:1775764678.280663:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23864fe300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42944:x1862022119027840:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764678.280684:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b2384b87800 lock: 000000004395017b/0x73cfd86cfc17b83d lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x115:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e03050c89 expref: -99 pid: 59614 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764678.280808:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764678.280815:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23864fe300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42944:x1862022119027840:12345-192.168.202.128@tcp:104: Request processed in 159us (14861us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764678.280823:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 179 00000100:00100000:1.0:1775764678.299639:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764678.534520:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764678.546904:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764678.573363:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764678.588069:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000002:00100000:1.0:1775764678.618574:0:59614:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fdc00 x1862022098506368/t34359738946(34359738946) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 608/664 e 0 to 0 dl 1775764694 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764678.627858:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7e6eda44-5508-487e-b17f-ff28ad9d3512:2376:1862022098511104:192.168.202.128@tcp:3:lfs.0 00000080:00200000:1.0:1775764678.712808:0:59616:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764678.712845:0:59616:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764678.712865:0:59616:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764678.712882:0:59616:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764678.712891:0:59616:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764678.712900:0:59616:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764678.712905:0:59616:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b859 00000080:00010000:1.0:1775764678.712922:0:59616:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b859 from it ffffa60b01efbb58 00010000:00010000:1.0:1775764678.712926:0:59616:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764678.712936:0:59616:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764678.712951:0:59616:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764678.712959:0:59616:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21i.sanity-pcc, flags=8 00000080:00200008:1.0:1775764678.712968:0:59616:0:(file.c:6226:ll_getattr_dentry()) START file f21i.sanity-pcc:[0x200003ab1:0x37:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764678.712973:0:59616:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab1:0x37:0x0](ffff8b238990cb08),name=f21i.sanity-pcc 00000002:00010000:1.0:1775764678.712986:0:59616:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x37:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764678.712994:0:59616:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb600/0x73cfd86cfc17b860 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050d0e expref: -99 pid: 59615 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764678.713003:0:59616:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb600/0x73cfd86cfc17b860 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050d0e expref: -99 pid: 59615 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764678.713010:0:59616:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764678.713015:0:59616:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x37:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764678.713018:0:59616:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x37:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17b860 00000080:00010000:1.0:1775764678.713022:0:59616:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b860 from it ffffa60b01efbcb8 00010000:00010000:1.0:1775764678.713034:0:59616:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb600/0x73cfd86cfc17b860 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03050d0e expref: -99 pid: 59615 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764678.713042:0:59616:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb600/0x73cfd86cfc17b860 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050d0e expref: -99 pid: 59615 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764678.713076:0:59616:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764678.713083:0:59616:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764678.713089:0:59616:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764678.713097:0:59616:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764678.713102:0:59616:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764678.713106:0:59616:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764678.713109:0:59616:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b859 00000080:00010000:1.0:1775764678.713112:0:59616:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b859 from it ffffa60b01efbb98 00010000:00010000:1.0:1775764678.713115:0:59616:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764678.713132:0:59616:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764678.713140:0:59616:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764678.713145:0:59616:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21i.sanity-pcc, flags=8 00000080:00200008:1.0:1775764678.713212:0:59616:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764678.713216:0:59616:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764678.713222:0:59616:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764678.713228:0:59616:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764678.713235:0:59616:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764678.713240:0:59616:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764678.713244:0:59616:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764678.713246:0:59616:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b859 00000080:00010000:1.0:1775764678.713250:0:59616:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b859 from it ffffa60b01efbcb8 00010000:00010000:1.0:1775764678.713253:0:59616:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764678.713259:0:59616:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764678.713274:0:59616:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764678.713279:0:59616:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764678.713284:0:59616:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764678.713292:0:59616:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764678.713298:0:59616:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764678.713302:0:59616:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764678.713304:0:59616:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b859 00000080:00010000:1.0:1775764678.713307:0:59616:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b859 from it ffffa60b01efbb58 00010000:00010000:1.0:1775764678.713309:0:59616:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764678.713315:0:59616:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764678.713321:0:59616:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764678.713325:0:59616:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21i.sanity-pcc, flags=8 00000080:00200008:1.0:1775764678.713329:0:59616:0:(file.c:6226:ll_getattr_dentry()) START file f21i.sanity-pcc:[0x200003ab1:0x37:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764678.713332:0:59616:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab1:0x37:0x0](ffff8b238990cb08),name=f21i.sanity-pcc 00000002:00010000:1.0:1775764678.713339:0:59616:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x37:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764678.713345:0:59616:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb600/0x73cfd86cfc17b860 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050d0e expref: -99 pid: 59615 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764678.713352:0:59616:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb600/0x73cfd86cfc17b860 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050d0e expref: -99 pid: 59615 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764678.713357:0:59616:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764678.713361:0:59616:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x37:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764678.713363:0:59616:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x37:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17b860 00000080:00010000:1.0:1775764678.713366:0:59616:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b860 from it ffffa60b01efbcb8 00010000:00010000:1.0:1775764678.713379:0:59616:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb600/0x73cfd86cfc17b860 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03050d0e expref: -99 pid: 59615 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764678.713386:0:59616:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eb600/0x73cfd86cfc17b860 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050d0e expref: -99 pid: 59615 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:1.0:1775764678.713903:0:59616:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764678.713908:0:59616:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764678.713916:0:59616:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764678.713924:0:59616:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764678.713934:0:59616:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764678.713940:0:59616:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764678.713944:0:59616:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764678.713946:0:59616:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b859 00000080:00010000:1.0:1775764678.713949:0:59616:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b859 from it ffffa60b01efbcb8 00010000:00010000:1.0:1775764678.713952:0:59616:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764678.713958:0:59616:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764678.714003:0:59616:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764678.714011:0:59616:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764678.714017:0:59616:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764678.714039:0:59616:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764678.714051:0:59616:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764678.714057:0:59616:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764678.714059:0:59616:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b859 00000080:00010000:1.0:1775764678.714062:0:59616:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b859 from it ffffa60b01efbbb0 00010000:00010000:1.0:1775764678.714064:0:59616:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764678.714071:0:59616:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764678.714079:0:59616:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764678.714086:0:59616:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 304000 00000080:00200080:1.0:1775764678.714172:0:59616:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=401866d9 arg=1c4f2a0 00000002:00010000:1.0:1775764678.714195:0:59616:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x37:0x0]) in obj [0x200003ab1:0x37:0x0], intent: open flags 03004040000002 00010000:00010000:1.0:1775764678.714254:0:59616:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740c00/0x73cfd86cfc17b867 lrc: 3/1,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 59616 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.714262:0:59616:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x81000000001000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740c00/0x73cfd86cfc17b867 lrc: 3/1,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 59616 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764678.714274:0:59616:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764678.714286:0:59616:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fdc00 x1862022098512384/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764678.714296:0:59616:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740c00/0x73cfd86cfc17b867 lrc: 3/1,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x81000000000000 pid: 59616 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764678.714321:0:59616:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59616:1862022098512384:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764678.714549:0:59616:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e1240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764678.724258:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764678.743719:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764678.743854:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200000:1.0:1775764678.781797:0:59617:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764678.781847:0:59617:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764678.781877:0:59617:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764678.781899:0:59617:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764678.781909:0:59617:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764678.782834:0:59617:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764678.782839:0:59617:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b859 00000080:00010000:1.0:1775764678.782850:0:59617:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b859 from it ffffa60b01eebbf8 00010000:00010000:1.0:1775764678.782856:0:59617:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764678.782869:0:59617:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764678.782894:0:59617:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764678.782930:0:59617:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f21i.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), file ffff8b239847ef00, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764678.782938:0:59617:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f21i.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=open 00000002:00010000:1.0:1775764678.782956:0:59617:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f21i.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764678.783044:0:59617:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741400/0x73cfd86cfc17b86e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 59617 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.783052:0:59617:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741400/0x73cfd86cfc17b86e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 59617 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764678.783066:0:59617:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764678.783079:0:59617:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fc380 x1862022098513408/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764678.783089:0:59617:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741400/0x73cfd86cfc17b86e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 59617 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764678.783126:0:59617:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59617:1862022098513408:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764678.783497:0:59617:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e1480 going to sleep for 11 seconds 00000080:00200008:1.0:1775764678.892355:0:59624:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764678.892382:0:59624:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764678.892406:0:59624:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764678.892428:0:59624:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764678.892444:0:59624:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764678.892659:0:59624:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764678.892668:0:59624:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764678.892672:0:59624:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b859 00000080:00010000:1.0:1775764678.892679:0:59624:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b859 from it ffffa60b01e73cb8 00010000:00010000:1.0:1775764678.892683:0:59624:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764678.892697:0:59624:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764678.892726:0:59624:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764678.892733:0:59624:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764678.892740:0:59624:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764678.892748:0:59624:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764678.892753:0:59624:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764678.892757:0:59624:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764678.892759:0:59624:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b859 00000080:00010000:1.0:1775764678.892763:0:59624:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b859 from it ffffa60b01e73b58 00010000:00010000:1.0:1775764678.892766:0:59624:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764678.892773:0:59624:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00000080:00200000:1.0:1775764678.892781:0:59624:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764678.892794:0:59624:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=f21i.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags=8 00000080:00200000:1.0:1775764678.892798:0:59624:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f21i.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec3a88), intent=getattr 00000002:00010000:1.0:1775764678.892809:0:59624:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f21i.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764678.892858:0:59624:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 59624 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.892867:0:59624:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 59624 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.892875:0:59624:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 59624 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764678.892902:0:59624:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59624:1862022098513792:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764678.893145:0:59624:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e1840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764678.898729:0:59624:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59624:1862022098513792:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764678.898768:0:59624:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.898792:0:59624:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 4/1,0 mode: --/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.898799:0:59624:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 4/1,0 mode: --/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.898811:0:59624:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.898824:0:59624:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.898837:0:59624:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 5/2,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.898850:0:59624:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 5/2,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.898858:0:59624:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 4/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764678.898870:0:59624:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864ff800 x1862022098513792/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 584/840 e 0 to 0 dl 1775764694 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764678.898883:0:59624:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764678.898897:0:59624:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.898919:0:59624:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 4 res 8589949617/55 (0 0) 00000080:00200000:1.0:1775764678.898928:0:59624:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764678.899044:0:59624:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x37:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764678.899057:0:59624:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab1:0x37:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764678.899060:0:59624:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1775764678.899068:0:59624:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764678.899283:0:59624:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x37:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764678.899363:0:59624:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab1:0x37:0x0], updating i_size 20 00000080:00200000:1.0:1775764678.899449:0:59624:0:(namei.c:147:ll_iget()) got inode: [0x200003ab1:0x37:0x0](ffff8b238990cb08): rc = 0 00000080:00200000:1.0:1775764678.899524:0:59624:0:(file.c:6766:ll_layout_conf()) [0x200003ab1:0x37:0x0]: layout version change: 4294967294 -> 4 00000080:00010000:1.0:1775764678.899536:0:59624:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x37:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17b875 00000080:00010000:1.0:1775764678.899549:0:59624:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x37:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764678.899553:0:59624:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b875 from it ffffa60b01e73b20 00010000:00010000:1.0:1775764678.899556:0:59624:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.899565:0:59624:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:1.0:1775764678.899606:0:59624:0:(file.c:6226:ll_getattr_dentry()) START file f21i.sanity-pcc:[0x200003ab1:0x37:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764678.899612:0:59624:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab1:0x37:0x0](ffff8b238990cb08),name=f21i.sanity-pcc 00000002:00010000:1.0:1775764678.899632:0:59624:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x37:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764678.899641:0:59624:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764678.899652:0:59624:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 2/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764678.899658:0:59624:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764678.899662:0:59624:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab1:0x37:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764678.899665:0:59624:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x37:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17b875 00000080:00010000:1.0:1775764678.899668:0:59624:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b875 from it ffffa60b01e73cb8 00010000:00010000:1.0:1775764678.899671:0:59624:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 3/1,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764678.899677:0:59624:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764678.899728:0:59624:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764678.899735:0:59624:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764678.899742:0:59624:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764678.899749:0:59624:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00000080:00200000:1.0:1775764678.899755:0:59624:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764678.899758:0:59624:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764678.899761:0:59624:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b859 00000080:00010000:1.0:1775764678.899764:0:59624:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b859 from it ffffa60b01e73bf8 00010000:00010000:1.0:1775764678.899767:0:59624:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764678.899774:0:59624:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23c2044400/0x73cfd86cfc17b859 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050cf2 expref: -99 pid: 59615 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 0 00000080:00200000:1.0:1775764678.899781:0:59624:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764678.899787:0:59624:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f21i.sanity-pcc, flags=265 00000080:00200000:1.0:1775764678.899793:0:59624:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab1:0x37:0x0](ffff8b238990cb08), inode mode 81a4 mask 44 00000080:00200008:1.0:1775764678.899801:0:59624:0:(file.c:1057:ll_file_open()) START file f21i.sanity-pcc:[0x200003ab1:0x37:0x0](ffff8b238990cb08), flags 100000 00000002:00010000:1.0:1775764678.899813:0:59624:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x37:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 04240100001 00010000:00010000:1.0:1775764678.899859:0:59624:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b87c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 59624 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.899867:0:59624:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b87c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 59624 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764678.899879:0:59624:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764678.899887:0:59624:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ff800 x1862022098514048/t0(0) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764678.899899:0:59624:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b87c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 59624 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764678.899993:0:59624:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59624:1862022098514048:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764678.900275:0:59624:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e1840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764678.905849:0:59624:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59624:1862022098514048:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764678.905887:0:59624:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b87c lrc: 4/1,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050da8 expref: -99 pid: 59624 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.905898:0:59624:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b87c lrc: 4/1,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050da8 expref: -99 pid: 59624 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.905909:0:59624:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b87c lrc: 4/1,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050da8 expref: -99 pid: 59624 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.905921:0:59624:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b87c lrc: 4/1,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050da8 expref: -99 pid: 59624 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764678.905935:0:59624:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23864ff800 x1862022098514048/t34359738970(34359738970) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764694 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764678.905949:0:59624:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864ff800 x1862022098514048/t34359738970(34359738970) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764694 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764678.905964:0:59624:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b87c lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050da8 expref: -99 pid: 59624 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764678.905978:0:59624:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b87c lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050da8 expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.905990:0:59624:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b239824a400 type 13 mode 16 res 8589949617/55 (0 0) 00000080:00200000:1.0:1775764678.905998:0:59624:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764678.906104:0:59624:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab1:0x37:0x0], updating i_size 20 00000080:00200000:1.0:1775764678.906137:0:59624:0:(file.c:6766:ll_layout_conf()) [0x200003ab1:0x37:0x0]: layout version change: 4 -> 4 00000080:00010000:1.0:1775764678.906150:0:59624:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab1:0x37:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17b87c 00000080:00010000:1.0:1775764678.906158:0:59624:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b87c from it ffffa60b01e73bf0 00010000:00010000:1.0:1775764678.906162:0:59624:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b87c lrc: 3/1,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03050da8 expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.906171:0:59624:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b87c lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03050da8 expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764678.906196:0:59624:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864ff800 x1862022098514048/t34359738970(34359738970) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764694 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764678.906277:0:59624:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab1:0x37:0x0](ffff8b238990cb08) cmd=400866fc arg=7ffd7d1a52f8 00800000:00000080:1.0:1775764678.906309:0:59624:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=401866d9 len=56 karg=ffff8b23891eda00 uarg=0000000000000000 00000002:00000080:1.0:1775764678.906317:0:59624:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2384b87800: cmd=401866d9 len=56 karg=ffff8b23891eda00 uarg=0000000000000000 00010000:00010000:1.0:1775764678.906339:0:59624:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b87c lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03050da8 expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764678.906357:0:59624:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab1:0x37:0x0]: losing layout lock 00010000:00010000:1.0:1775764678.906374:0:59624:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764678.906388:0:59624:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab1:0x37:0x0]: losing layout lock 00010000:00010000:1.0:1775764678.906410:0:59624:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b87c lrc: 1/0,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03050da8 expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764678.906417:0:59624:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 1/0,0 mode: --/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764678.906423:0:59624:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00000100:00100000:1.0:1775764678.906441:0:59624:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc700 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59624:1862022098514304:192.168.202.128@tcp:103:lfs.0 00000100:00100000:1.0:1775764678.906707:0:59624:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e1840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764678.910297:0:59624:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc700 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59624:1862022098514304:192.168.202.128@tcp:103:lfs.0 00010000:00010000:1.0:1775764678.910325:0:59624:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8740000/0x73cfd86cfc17b87c lrc: 0/0,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03050da8 expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764678.910341:0:59624:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741800/0x73cfd86cfc17b875 lrc: 0/0,0 mode: --/PR res: [0x200003ab1:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03050d9a expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:1.0:1775764678.910363:0:59624:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 58, max 7 00000100:00100000:1.0:1775764678.910372:0:59624:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ffb80 x1862022098514432/t0(0) o58->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764678.910401:0:59624:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59624:1862022098514432:192.168.202.128@tcp:58:lfs.0 00000100:00100000:1.0:1775764678.910613:0:59624:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e1840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764678.918099:0:59624:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59624:1862022098514432:192.168.202.128@tcp:58:lfs.0 00000080:00010000:1.0:1775764678.918178:0:59624:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab1:0x37:0x0](ffff8b238990cb08) 00000002:00010000:1.0:1775764678.918193:0:59624:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab1:0x37:0x0]) in obj [0x200003ab1:0x37:0x0], intent: layout flags 0 00010000:00010000:1.0:1775764678.918265:0:59624:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b883 lrc: 3/1,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 59624 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.918281:0:59624:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b883 lrc: 3/1,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 59624 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.918294:0:59624:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b883 lrc: 3/1,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 59624 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764678.918323:0:59624:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59624:1862022098514560:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764678.918583:0:59624:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e1840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764678.922659:0:59624:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59624:1862022098514560:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764678.922693:0:59624:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b883 lrc: 4/1,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050dbd expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764678.922706:0:59624:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b883 lrc: 4/1,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050dbd expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764679.661037:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119029760 00000100:00100000:1.0:1775764679.661602:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119029760 00000100:00100000:1.0:1775764679.661775:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 188 00000100:00100000:1.0:1775764679.661789:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dac380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42356:x1862022119029760:12345-192.168.202.128@tcp:107:hsm_cdtr.0 00000100:00100000:1.0:1775764679.661930:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dac380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42356:x1862022119029760:12345-192.168.202.128@tcp:107:hsm_cdtr.0 Request processed in 145us (8184us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764679.661939:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 188 00000002:00100000:1.0:1775764679.722013:0:59625:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864f2d80 x1862022098515072/t34359738971(34359738971) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764695 ref 2 fl Complete:RQU/604/0 rc 301/301 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764679.722107:0:59625:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab1:0x37:0x0](ffff8b238990cb08), inode mode 81a4 mask 44 00000080:00200080:1.0:1775764679.722142:0:59625:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=c0086696 arg=7f38f0000d20 00800000:00000080:1.0:1775764679.722162:0:59625:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2384b87800: cmd=c0086696 len=4128 karg=ffff8b23ab52a000 uarg=0000000000000000 00000002:00000080:1.0:1775764679.722169:0:59625:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2384b87800: cmd=c0086696 len=4128 karg=ffff8b23ab52a000 uarg=0000000000000000 00000002:00000080:1.0:1775764679.722175:0:59625:0:(mdc_request.c:1738:mdc_ioc_fid2path()) path get [0x200003ab1:0x37:0x0] from 18446744073709551615 #0 00000100:00100000:1.0:1775764679.722212:0:59625:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc700 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:59625:1862022098515456:192.168.202.128@tcp:53:lhsmtool_posix.0 00000100:00100000:1.0:1775764679.722316:0:59625:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e1840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764679.840224:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764679.848685:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119030016 00000100:00100000:1.0:1775764679.848694:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119030016 00000100:00100000:1.0:1775764679.848741:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 190 00000100:00100000:1.0:1775764679.848750:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dae300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42311:x1862022119030016:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764679.848760:0:46139:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8bb lrc: 5/0,1 mode: --/PW res: [0x240000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03050e50 expref: -99 pid: 59625 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764679.848769:0:46139:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8bb lrc: 5/0,1 mode: --/PW res: [0x240000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03050e50 expref: -99 pid: 59625 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764679.848778:0:46139:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8bb lrc: 5/0,1 mode: --/PW res: [0x240000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e03050e50 expref: -99 pid: 59625 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764679.848787:0:46139:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8bb lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x57b8ff0e03050e50 expref: -99 pid: 59625 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764679.848796:0:46139:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8bb lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x57b8ff0e03050e50 expref: -99 pid: 59625 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764679.848810:0:46139:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8bb lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x57b8ff0e03050e50 expref: -99 pid: 59625 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764679.848841:0:46139:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b23ac6ebe00) 00000100:00100000:1.0:1775764679.848911:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dae300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42311:x1862022119030016:12345-192.168.202.128@tcp:105: Request processed in 150us (1123us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764679.848919:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 190 00000080:00200000:1.0:1775764679.862954:0:59625:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x3b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:1.0:1775764679.863037:0:59625:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:59625:1862022098518016:192.168.202.128@tcp:44:lhsmtool_posix.0 00000100:00100000:1.0:1775764679.863159:0:59625:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e1240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764679.917599:0:59625:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:7e6eda44-5508-487e-b17f-ff28ad9d3512:59625:1862022098518016:192.168.202.128@tcp:44:lhsmtool_posix.0 00000080:00200000:1.0:1775764679.917678:0:59625:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x3b:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764679.917694:0:59625:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:1.0:1775764679.971274:0:2377:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b2384b87800: committing for last_committed 34359738428 gen 2 00000100:00100000:1.0:1775764679.971382:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098518144:192.168.202.128@tcp:16:lhsmtool_posix.0 00000100:00100000:1.0:1775764679.991943:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764679.991993:0:46139:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764679.992025:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119030272 00000100:00100000:1.0:1775764679.992036:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119030272 00000100:00100000:1.0:1775764679.992092:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 191 00000100:00100000:1.0:1775764679.992103:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2390282850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42326:x1862022119030272:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764679.992124:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b89f lrc: 2/0,0 mode: PR/PR res: [0x200003ab1:0x3b:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03050e26 expref: -99 pid: 59625 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764679.992239:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764679.992247:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2390282850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42326:x1862022119030272:12345-192.168.202.128@tcp:104: Request processed in 146us (319us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764679.992255:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 191 00000100:00100000:1.0:1775764680.007247:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764680.039872:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119030528 00000100:00100000:1.0:1775764680.039894:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119030528 00000100:00100000:1.0:1775764680.040229:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 193 00000100:00100000:1.0:1775764680.040242:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dac380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42324:x1862022119030528:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764680.040271:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6ea800/0x73cfd86cfc17b8a6 lrc: 2/0,0 mode: CR/CR res: [0x200003ab1:0x3b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03050e34 expref: -99 pid: 59625 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764680.040553:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764680.040561:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dac380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42324:x1862022119030528:12345-192.168.202.128@tcp:104: Request processed in 322us (12600us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764680.040570:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 193 00000100:00100000:1.0:1775764680.068662:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119030912 00000100:00100000:1.0:1775764680.068674:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119030912 00000100:00100000:1.0:1775764680.068726:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 194 00000100:00100000:1.0:1775764680.068735:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dae300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42324:x1862022119030912:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764680.068752:0:46140:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b883 lrc: 5/1,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050dbd expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764680.068764:0:46140:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b883 lrc: 5/1,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050dbd expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764680.068778:0:46140:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b883 lrc: 5/1,0 mode: --/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050dbd expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764680.068787:0:46140:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b883 lrc: 5/1,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e03050dbd expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764680.068796:0:46140:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b883 lrc: 6/1,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e03050dbd expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764680.068804:0:46140:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23b8741600/0x73cfd86cfc17b883 lrc: 6/1,0 mode: CR/CR res: [0x200003ab1:0x37:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x80000000000 nid: local remote: 0x57b8ff0e03050dbd expref: -99 pid: 59624 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764680.068833:0:46140:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b23b8741600) 00000100:00100000:1.0:1775764680.068896:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dae300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:7e6eda44-5508-487e-b17f-ff28ad9d3512+5:42324:x1862022119030912:12345-192.168.202.128@tcp:105: Request processed in 162us (343us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764680.068910:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 194 00000020:00010000:1.0:1775764680.102196:0:59624:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b5d8 (ffff8b23812eca28/0) at cl_lock_release():245 00000020:00010000:1.0:1775764680.102201:0:59624:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b5d8 (ffff8b23812eca28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764680.102205:0:59624:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24d30 (ffff8b23812ec5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764680.102217:0:59624:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8c9 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03050e96 expref: -99 pid: 59624 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764680.102235:0:59624:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8c9 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03050e96 expref: -99 pid: 59624 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:1.0:1775764680.102261:0:59624:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b5d8 (ffff8b23812eca28/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764680.102264:0:59624:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24d30 (ffff8b23812eca28/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764680.102273:0:59624:0:(vvp_io.c:517:vvp_io_advance()) advancing 20 bytes 00000080:00200000:1.0:1775764680.102304:0:59624:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x37:0x0] ignore/verify layout 0/0, layout version 5 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764680.102309:0:59624:0:(file.c:2206:ll_file_io_generic()) f21i.sanity-pcc: 1 io complete with rc: 0, result: 20, restart: 0 00000080:00200000:1.0:1775764680.102315:0:59624:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 20 00000080:00200008:1.0:1775764680.102581:0:59624:0:(file.c:2454:do_file_read_iter()) START file f21i.sanity-pcc:[0x200003ab1:0x37:0x0], ppos: 20, count: 1048576 00000080:00200000:1.0:1775764680.102610:0:59624:0:(file.c:2058:ll_file_io_generic()) f21i.sanity-pcc: read ppos: 20, bytes: 1048576 00000080:00200000:1.0:1775764680.102615:0:59624:0:(file.c:1815:ll_io_set_mirror()) f21i.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764680.102623:0:59624:0:(cl_io.c:214:cl_io_rw_init()) header@0000000017fde005[0x110, 2, [0x200003ab1:0x37:0x0] hash] 00000020:00200000:1.0:1775764680.102625:0:59624:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [20, 1048596) 0 0 00000080:00200000:1.0:1775764680.102629:0:59624:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x37:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764680.102652:0:59624:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 20 chunk: [20, 1048596) 1048596, 1048576 00020000:00200000:1.0:1775764680.102657:0:59624:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764680.102686:0:59624:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 256] 00000008:00010000:1.0:1775764680.102699:0:59624:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764680.102703:0:59624:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318a30, osc lock ffff8b2382cf7c80, flags 0x0 00010000:00010000:1.0:1775764680.102725:0:59624:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8c9 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x116:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03050e96 expref: -99 pid: 59624 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764680.102739:0:59624:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1052671) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8c9 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03050e96 expref: -99 pid: 59624 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764680.102753:0:59624:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8c9 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03050e96 expref: -99 pid: 59624 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764680.102762:0:59624:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8c9 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03050e96 expref: -99 pid: 59624 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764680.102771:0:59624:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8c9 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03050e96 expref: -99 pid: 59624 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764680.102780:0:59624:0:(vvp_io.c:826:vvp_io_read_start()) f21i.sanity-pcc: read [20, 1048596) 00000001:00010000:1.0:1775764680.102800:0:59624:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab1:0x37:0x0] 00000008:00010000:1.0:1775764680.102806:0:59624:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764680.102808:0:59624:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b238a318630, osc lock ffff8b2382cf7300, flags 0x400001002 00010000:00010000:1.0:1775764680.102816:0:59624:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8c9 lrc: 4/2,0 mode: PR/PR res: [0x240000400:0x116:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03050e96 expref: -99 pid: 59624 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764680.102824:0:59624:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8c9 lrc: 4/2,0 mode: PR/PR res: [0x240000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03050e96 expref: -99 pid: 59624 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1775764680.102832:0:59624:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764680.102835:0:59624:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8c9 lrc: 7/2,0 mode: PR/PR res: [0x240000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03050e96 expref: -99 pid: 59624 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764680.102843:0:59624:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8c9 lrc: 6/1,0 mode: PR/PR res: [0x240000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03050e96 expref: -99 pid: 59624 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1775764680.102851:0:59624:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764680.102858:0:59624:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab1:0x37:0x0] updating i_size 20 i_blocks 11 00000020:00010000:1.0:1775764680.102862:0:59624:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f18b68 (ffff8b2387cba5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764680.102864:0:59624:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f18b68 (ffff8b2387cba5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764680.102866:0:59624:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318630 (ffff8b2387cbacf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764680.102869:0:59624:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f18b68 (ffff8b2387cba5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764680.102872:0:59624:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318630 (ffff8b2387cba5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764680.102878:0:59624:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:20 00000020:00010000:1.0:1775764680.102883:0:59624:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb7c68 (ffff8b2387cba5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764680.102885:0:59624:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb7c68 (ffff8b2387cba5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764680.102888:0:59624:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a318a30 (ffff8b2387cbacf8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764680.102892:0:59624:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8c9 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03050e96 expref: -99 pid: 59624 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764680.102997:0:59624:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2384b87800 lock: 00000000d053785e/0x73cfd86cfc17b8c9 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03050e96 expref: -99 pid: 59624 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00010000:1.0:1775764680.103261:0:59624:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb7c68 (ffff8b2387cba5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764680.103267:0:59624:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a318a30 (ffff8b2387cba5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764680.103274:0:59624:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:1.0:1775764680.103299:0:59624:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x37:0x0] ignore/verify layout 0/0, layout version 5 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764680.103303:0:59624:0:(file.c:2206:ll_file_io_generic()) f21i.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:1.0:1775764680.103305:0:59624:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:1.0:1775764680.104037:0:59624:0:(file.c:447:ll_file_release()) START file f21i.sanity-pcc:[0x200003ab1:0x37:0x0](ffff8b238990cb08), flags 100000 00000002:00100000:1.0:1775764680.104123:0:59624:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864ff800 x1862022098514048/t34359738970(34359738970) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764694 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764680.104139:0:59624:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764680.104144:0:59624:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fe300 x1862022098520704/t0(0) o35->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764680.104173:0:59624:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59624:1862022098520704:192.168.202.128@tcp:35:lfs.0 00000100:00100000:1.0:1775764680.104699:0:59624:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e16c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764680.108208:0:59624:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job lfs:7e6eda44-5508-487e-b17f-ff28ad9d3512:59624:1862022098520704:192.168.202.128@tcp:35:lfs.0 00000002:00100000:1.0:1775764680.108258:0:59624:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864ff800 x1862022098514048/t34359738970(34359738970) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764694 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764680.268891:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764686.391915:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098522880:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764686.392036:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098523136:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764686.392076:0:2377:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:1.0:1775764686.392083:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098522752:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764686.392106:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098523008:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764686.392126:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098523264:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764686.401673:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098522752:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764686.401696:0:2377:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b2384b87800: committing for last_committed 34359738430 gen 2 00000100:00100000:1.0:1775764686.401720:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098523008:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764686.401734:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098523264:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764686.401743:0:2377:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2384b87800: committing for last_committed 34359738985 gen 1 00000100:00100000:1.0:1775764686.401768:0:2377:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23b9ea6d80 x1862022098521728/t34359738985(34359738985) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764696 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764686.401778:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098522880:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764686.401786:0:2377:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b2384b87800: committing for last_committed 34359738433 gen 1 00000100:00100000:1.0:1775764686.401794:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7e6eda44-5508-487e-b17f-ff28ad9d3512:2377:1862022098523136:192.168.202.128@tcp:400:kworker.0 00000080:00200008:1.0:1775764686.850298:0:60066:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764686.850316:0:60066:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764686.850348:0:60066:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764686.850373:0:60066:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764686.850391:0:60066:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764686.850399:0:60066:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764686.850410:0:60066:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764686.850414:0:60066:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b8d0 00000080:00010000:1.0:1775764686.850423:0:60066:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b8d0 from it ffffa60b0106fcb8 00010000:00010000:1.0:1775764686.850428:0:60066:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764686.850438:0:60066:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200008:1.0:1775764686.851437:0:60066:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764686.851445:0:60066:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764686.851487:0:60066:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764686.851501:0:60066:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764686.851515:0:60066:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764686.851529:0:60066:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764686.851535:0:60066:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764686.851539:0:60066:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b8d0 00000080:00010000:1.0:1775764686.851543:0:60066:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b8d0 from it ffffa60b0106fcb8 00010000:00010000:1.0:1775764686.851546:0:60066:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764686.851554:0:60066:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764686.851664:0:60066:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764686.851672:0:60066:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764686.851678:0:60066:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764686.851686:0:60066:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764686.851692:0:60066:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764686.851696:0:60066:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764686.851699:0:60066:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b8d0 00000080:00010000:1.0:1775764686.851702:0:60066:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b8d0 from it ffffa60b0106fbb0 00010000:00010000:1.0:1775764686.851705:0:60066:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764686.851712:0:60066:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764686.851722:0:60066:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764686.851730:0:60066:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 504000 00000080:00200080:1.0:1775764686.851746:0:60066:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=8080667f arg=7ffd44019f20 00000080:00200008:1.0:1775764686.851759:0:60066:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 504000 00000080:00200000:1.0:1775764686.852071:0:60066:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764686.852079:0:60066:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764686.852100:0:60066:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764686.852110:0:60066:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764686.852117:0:60066:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764686.852122:0:60066:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764686.852124:0:60066:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b8d0 00000080:00010000:1.0:1775764686.852128:0:60066:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b8d0 from it ffffa60b0106fbb0 00010000:00010000:1.0:1775764686.852131:0:60066:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764686.852139:0:60066:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764686.852148:0:60066:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764686.852151:0:60066:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 504000 00000080:00200080:1.0:1775764686.852159:0:60066:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=8080667f arg=7ffd44018e00 00000080:00200008:1.0:1775764686.852165:0:60066:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 504000 00000080:00200000:1.0:1775764686.852265:0:60066:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764686.852273:0:60066:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764686.852279:0:60066:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764686.852287:0:60066:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764686.852293:0:60066:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764686.852298:0:60066:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764686.852300:0:60066:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b8d0 00000080:00010000:1.0:1775764686.852304:0:60066:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b8d0 from it ffffa60b0106fbb0 00010000:00010000:1.0:1775764686.852307:0:60066:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764686.852314:0:60066:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00000080:00200000:1.0:1775764686.852323:0:60066:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:1.0:1775764686.852327:0:60066:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 100000 00000080:00200080:1.0:1775764686.852893:0:60066:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) cmd=c01466fc arg=7ffd44016ce0 00000080:00200008:1.0:1775764686.853048:0:60066:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), flags 100000 00000080:00200008:1.0:1775764687.178038:0:60083:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764687.178055:0:60083:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764687.178087:0:60083:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764687.178109:0:60083:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764687.178128:0:60083:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00000080:00200000:1.0:1775764687.178134:0:60083:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764687.178145:0:60083:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764687.178149:0:60083:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b8d0 00000080:00010000:1.0:1775764687.178155:0:60083:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b8d0 from it ffffa60b0146bcb8 00010000:00010000:1.0:1775764687.178158:0:60083:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00010000:00010000:1.0:1775764687.178168:0:60083:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 0 00000080:00200008:1.0:1775764687.179091:0:60083:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764687.179098:0:60083:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:1.0:1775764687.179114:0:60083:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764687.179193:0:60083:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740000/0x73cfd86cfc17b8ec lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 60083 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764687.179203:0:60083:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740000/0x73cfd86cfc17b8ec lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 60083 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764687.179215:0:60083:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740000/0x73cfd86cfc17b8ec lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 60083 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764687.179241:0:60083:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job lsof:82781e18-e0e3-4f9f-9a2b-041bf0344414:60083:1862022098523776:192.168.202.128@tcp:101:lsof.0 00010000:00010000:1.0:1775764687.671687:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764689.726935:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1775764696.458901:0:60348:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764696.458917:0:60348:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764696.458948:0:60348:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764696.458971:0:60348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 0 00010000:00010000:1.0:1775764696.458990:0:60348:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 0 00000080:00200000:1.0:1775764696.458998:0:60348:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764696.459007:0:60348:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764696.459011:0:60348:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b8d0 00000080:00010000:1.0:1775764696.459018:0:60348:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b8d0 from it ffffa60b01dcbcb8 00010000:00010000:1.0:1775764696.459023:0:60348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 0 00010000:00010000:1.0:1775764696.459031:0:60348:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200008:1.0:1775764696.459183:0:60348:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764696.459188:0:60348:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88),name=/ 00000002:00010000:1.0:1775764696.459195:0:60348:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764696.459202:0:60348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764696.459212:0:60348:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:1.0:1775764696.459225:0:60348:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764696.459230:0:60348:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764696.459232:0:60348:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17b8d0 00000080:00010000:1.0:1775764696.459646:0:60348:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b8d0 from it ffffa60b01dcbcb8 00010000:00010000:1.0:1775764696.459651:0:60348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764696.459660:0:60348:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2384b87800 lock: ffff8b23ac6eaa00/0x73cfd86cfc17b8d0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050ea4 expref: -99 pid: 59626 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1775764696.487219:0:60348:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec3a88) 00000080:00200000:1.0:1775764696.487266:0:60348:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000002:0x1:0x0](ffff8b23899080c8) 00000080:00200000:1.0:1775764696.487323:0:60348:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200000002:0x2:0x0](ffff8b238990e3c8) 00000080:00200000:1.0:1775764696.487389:0:60348:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab1:0x22:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764696.487400:0:60348:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab1:0x22:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764696.487427:0:60348:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200003ab1:0x22:0x0](ffff8b23899c4b08) 00000002:00100000:1.0:1775764696.487545:0:60348:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea4000 x1862022098401536/t34359738747(34359738747) o101->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764536 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764696.487559:0:60348:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2384b87800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764696.487564:0:60348:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fed80 x1862022098524672/t0(0) o35->lustre-MDT0000-mdc-ffff8b2384b87800@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764696.487595:0:60348:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job umount:7e6eda44-5508-487e-b17f-ff28ad9d3512:60348:1862022098524672:192.168.202.128@tcp:35:umount.0 00000100:00100000:1.0:1775764696.487896:0:60348:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f3c0 going to sleep for 11 seconds 00000020:00000080:1.0:1775764696.636588:0:8:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff8b238939d000 for lustre-MDT0000-mdc-ffff8b2384b87800 00000020:00000080:1.0:1775764696.636616:0:8:0:(genops.c:904:class_export_put()) final put ffff8b2389399000/7e6eda44-5508-487e-b17f-ff28ad9d3512 00000020:00000080:1.0:1775764696.636624:0:8:0:(genops.c:863:class_export_destroy()) destroying export ffff8b2389399000/7e6eda44-5508-487e-b17f-ff28ad9d3512 for lustre-MDT0000-mdc-ffff8b2384b87800 00000020:01000000:1.0:1775764696.636640:0:8:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-MDT0000-mdc-ffff8b2384b87800 (7e6eda44-5508-487e-b17f-ff28ad9d3512) 00000100:00100000:1.0:1775764696.958058:0:60358:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadc00 pname:cluuid:pid:xid:nid:opc:job mount.lustre:24b54882-c0a1-43f2-954f-009e5a9afc9d:60358:1862022098531200:192.168.202.128@tcp:503:mount.lustre.0 00000100:00100000:1.0:1775764696.958546:0:60362:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job llog_process_th:24b54882-c0a1-43f2-954f-009e5a9afc9d:60362:1862022098531328:192.168.202.128@tcp:502:llog_process_th.0 00000100:00100000:1.0:1775764696.958634:0:60362:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394fb40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764696.968515:0:60362:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job llog_process_th:24b54882-c0a1-43f2-954f-009e5a9afc9d:60362:1862022098531328:192.168.202.128@tcp:502:llog_process_th.0 00000020:01000000:1.0:1775764696.968559:0:60362:0:(obd_config.c:1888:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:1.0:1775764696.968574:0:60362:0:(obd_config.c:1468:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1775764696.968576:0:60362:0:(obd_config.c:1559:class_process_config()) marker 4 (0x1) lustre-clilov lov setup 00000020:01000000:1.0:1775764696.968583:0:60362:0:(obd_config.c:2003:class_config_llog_handler()) cmd cf001, instance name: lustre-clilov-ffff8b2388a49800 00000020:00000080:1.0:1775764696.968586:0:60362:0:(obd_config.c:1468:class_process_config()) processing cmd: cf001 00000020:00000080:1.0:1775764696.968600:0:60362:0:(genops.c:412:class_newdev()) Allocate new device lustre-clilov-ffff8b2388a49800 (ffff8b23a00019c0) 00000020:00000080:1.0:1775764696.968623:0:60362:0:(obd_config.c:720:class_attach_name()) OBD: dev 1 attached type lov with refcount 2 00000020:01000000:1.0:1775764696.968630:0:60362:0:(obd_config.c:2003:class_config_llog_handler()) cmd cf003, instance name: lustre-clilov-ffff8b2388a49800 00000020:00000080:1.0:1775764696.968633:0:60362:0:(obd_config.c:1468:class_process_config()) processing cmd: cf003 00000100:00100000:1.0:1775764697.016770:0:60358:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad180 pname:cluuid:pid:xid:nid:opc:job mount.lustre:24b54882-c0a1-43f2-954f-009e5a9afc9d:60358:1862022098531584:192.168.202.128@tcp:101:mount.lustre.0 00010000:00010000:1.0:1775764697.016805:0:60358:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.128@tcp lock: ffff8b23c2045600/0x73cfd86cfc17b91d lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x57b8ff0e03050f4c expref: -99 pid: 60358 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764697.016819:0:60358:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.128@tcp lock: ffff8b23c2045600/0x73cfd86cfc17b91d lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x57b8ff0e03050f4c expref: -99 pid: 60358 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764697.017542:0:60358:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job mount.lustre:24b54882-c0a1-43f2-954f-009e5a9afc9d:60358:1862022098531712:192.168.202.128@tcp:256:mount.lustre.0 00000100:00100000:1.0:1775764697.017647:0:60358:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f900 going to sleep for 11 seconds 00000100:00100000:1.0:1775764697.070411:0:60372:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:24b54882-c0a1-43f2-954f-009e5a9afc9d:60372:1862022098532224:192.168.202.128@tcp:502:llog_process_th.0 00000100:00100000:1.0:1775764697.070728:0:60372:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394fc00 going to sleep for 11 seconds 00000100:00100000:1.0:1775764697.074314:0:60372:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:24b54882-c0a1-43f2-954f-009e5a9afc9d:60372:1862022098532224:192.168.202.128@tcp:502:llog_process_th.0 00000020:01000000:1.0:1775764697.074366:0:60372:0:(obd_config.c:1888:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:1.0:1775764697.074378:0:60372:0:(obd_config.c:1468:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1775764697.074381:0:60372:0:(obd_config.c:1559:class_process_config()) marker 2 (0x1) general debug_raw_pointers 00000020:01000000:1.0:1775764697.074390:0:60372:0:(obd_config.c:2003:class_config_llog_handler()) cmd ce032, instance name: general-ffff8b2388a49800 00000020:00000080:1.0:1775764697.074393:0:60372:0:(obd_config.c:1468:class_process_config()) processing cmd: ce032 00000020:01000000:1.0:1775764697.074402:0:60372:0:(obd_config.c:1888:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1775764697.074404:0:60372:0:(obd_config.c:1468:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1775764697.074406:0:60372:0:(obd_config.c:1559:class_process_config()) marker 2 (0x2) general debug_raw_pointers 00000020:01000000:1.0:1775764697.074408:0:60372:0:(obd_config.c:1888:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:1.0:1775764697.074411:0:60372:0:(obd_config.c:1468:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1775764697.074412:0:60372:0:(obd_config.c:1559:class_process_config()) marker 3 (0x1) general lod.*.mdt_hash 00000020:01000000:1.0:1775764697.074415:0:60372:0:(obd_config.c:2003:class_config_llog_handler()) cmd ce032, instance name: general-ffff8b2388a49800 00000020:00000080:1.0:1775764697.074417:0:60372:0:(obd_config.c:1468:class_process_config()) processing cmd: ce032 00000020:01000000:1.0:1775764697.074689:0:60372:0:(obd_config.c:1888:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1775764697.074694:0:60372:0:(obd_config.c:1468:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1775764697.074695:0:60372:0:(obd_config.c:1559:class_process_config()) marker 3 (0x2) general lod.*.mdt_hash 00000020:01000000:1.0:1775764697.074698:0:60372:0:(obd_config.c:1888:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:1.0:1775764697.074701:0:60372:0:(obd_config.c:1468:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1775764697.074702:0:60372:0:(obd_config.c:1559:class_process_config()) marker 4 (0x1) lustre mdt.lustre-MDT0000.hsm_control 00000020:01000000:1.0:1775764697.074705:0:60372:0:(obd_config.c:2003:class_config_llog_handler()) cmd ce032, instance name: lustre-ffff8b2388a49800 00000020:00000080:1.0:1775764697.074707:0:60372:0:(obd_config.c:1468:class_process_config()) processing cmd: ce032 00000100:00100000:1.0:1775764697.535097:0:60404:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job mkdir:d5685161-9db4-4c4c-ac8e-956c40da0bd9:60404:1862022098533376:192.168.202.128@tcp:101:mkdir.0 00010000:00010000:1.0:1775764697.535146:0:60404:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764697.535160:0:60404:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764697.535179:0:60404:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764697.535196:0:60404:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764697.535211:0:60404:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764697.535219:0:60404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764697.535227:0:60404:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764697.535238:0:60404:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea5500 x1862022098533376/t0(0) o101->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764713 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mkdir.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764697.535255:0:60404:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764697.535271:0:60404:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b23893d5000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1775764697.535279:0:60404:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764697.535311:0:60404:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:1.0:1775764697.535317:0:60404:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) for lock 0x73cfd86cfc17b94e 00000080:00010000:1.0:1775764697.535324:0:60404:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) 00000080:00010000:1.0:1775764697.535327:0:60404:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b94e from it ffffa60b0202fbe0 00010000:00010000:1.0:1775764697.535331:0:60404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764697.535340:0:60404:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764697.535371:0:60404:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908), inode mode 41ed mask 1 00000080:00200000:1.0:1775764697.535421:0:60404:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=d21j.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec0908), flags=1536 00000080:00200000:1.0:1775764697.535427:0:60404:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908),name=/ 00000002:00010000:1.0:1775764697.535445:0:60404:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764697.535489:0:60404:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764697.535501:0:60404:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764697.535508:0:60404:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764697.535512:0:60404:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) 00000080:00010000:1.0:1775764697.535514:0:60404:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) for lock 0x73cfd86cfc17b94e 00000080:00010000:1.0:1775764697.535518:0:60404:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b94e from it ffffa60b0202fce0 00010000:00010000:1.0:1775764697.535521:0:60404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764697.535528:0:60404:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000080:00200000:1.0:1775764697.535534:0:60404:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908), inode mode 41ed mask 3 00000080:00200000:1.0:1775764697.535541:0:60404:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908),name=/ 00000002:00010000:1.0:1775764697.535547:0:60404:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764697.535552:0:60404:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00010000:1.0:1775764697.535559:0:60404:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000080:00200000:1.0:1775764697.535565:0:60404:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764697.535568:0:60404:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) 00000080:00010000:1.0:1775764697.535570:0:60404:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) for lock 0x73cfd86cfc17b94e 00000080:00010000:1.0:1775764697.535573:0:60404:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b94e from it ffffa60b0202fdc8 00010000:00010000:1.0:1775764697.535576:0:60404:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00010000:1.0:1775764697.535584:0:60404:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea600/0x73cfd86cfc17b94e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f76 expref: -99 pid: 60404 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:1.0:1775764697.535591:0:60404:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908), inode mode 41ed mask 3 00000080:00200000:1.0:1775764697.535596:0:60404:0:(namei.c:2226:do_mkdir()) VFS Op:name=d21j.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2390ec0908) 00000100:00100000:1.0:1775764697.683008:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119035008 00000100:00100000:1.0:1775764697.683022:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119035008 00000100:00100000:1.0:1775764697.683103:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 196 00000100:00100000:1.0:1775764697.683116:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dace00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42326:x1862022119035008:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764697.683136:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740000/0x73cfd86cfc17b8ec lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03050f3e expref: -99 pid: 60083 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764697.683241:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764697.683247:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dace00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42326:x1862022119035008:12345-192.168.202.128@tcp:104: Request processed in 135us (342us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764697.683255:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 196 00000080:00200008:1.0:1775764697.940421:0:60434:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec0908), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764697.940442:0:60434:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908),name=/ 00000002:00010000:1.0:1775764697.940512:0:60434:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764697.940598:0:60434:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740e00/0x73cfd86cfc17b955 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 60434 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764697.940610:0:60434:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740e00/0x73cfd86cfc17b955 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 60434 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764697.940622:0:60434:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740e00/0x73cfd86cfc17b955 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 60434 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764697.940658:0:60434:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:d5685161-9db4-4c4c-ac8e-956c40da0bd9:60434:1862022098534016:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764697.940784:0:60434:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000080:00200000:1.0:1775764698.108807:0:60443:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908),name=/ 00000002:00010000:1.0:1775764698.108856:0:60443:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764698.109027:0:60443:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740e00/0x73cfd86cfc17b955 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f92 expref: -99 pid: 60434 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:1.0:1775764698.109047:0:60443:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740e00/0x73cfd86cfc17b955 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f92 expref: -99 pid: 60434 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000080:00200000:1.0:1775764698.109054:0:60443:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764698.109064:0:60443:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) 00000080:00010000:1.0:1775764698.109067:0:60443:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) for lock 0x73cfd86cfc17b955 00000080:00010000:1.0:1775764698.109075:0:60443:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b955 from it ffffa60b00d17bb0 00010000:00010000:1.0:1775764698.109078:0:60443:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740e00/0x73cfd86cfc17b955 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050f92 expref: -99 pid: 60434 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:1.0:1775764698.109087:0:60443:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740e00/0x73cfd86cfc17b955 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f92 expref: -99 pid: 60434 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200000:1.0:1775764698.109103:0:60443:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908), inode mode 41ed mask 44 00000080:00200008:1.0:1775764698.109111:0:60443:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec0908), flags 504000 00000080:00200080:1.0:1775764698.109143:0:60443:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908) cmd=8080667f arg=7ffc352f4570 00000080:00200008:1.0:1775764698.109157:0:60443:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b2390ec0908), flags 504000 00000080:00010000:1.0:1775764698.455567:0:60457:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab3:0x2:0x0](ffff8b2390ec3248) 00000080:00200008:1.0:1775764698.455596:0:60457:0:(file.c:1057:ll_file_open()) START file f21j.sanity-pcc:[0x200003ab3:0x2:0x0](ffff8b2390ec3248), flags 100301 00000002:00100000:1.0:1775764698.455636:0:60457:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864f1880 x1862022098535296/t34359738995(34359738995) o101->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:12/10 lens 624/696 e 0 to 0 dl 1775764714 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cp.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1775764698.455679:0:60457:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b986 from it ffff8b23ab59c420 00010000:00010000:1.0:1775764698.455689:0:60457:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b9c5fa00/0x73cfd86cfc17b986 lrc: 3/1,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051041 expref: -99 pid: 60457 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764698.455701:0:60457:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b9c5fa00/0x73cfd86cfc17b986 lrc: 2/0,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051041 expref: -99 pid: 60457 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764698.455973:0:60457:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab3:0x2:0x0](ffff8b2390ec3248), inode mode 81a4 mask 40 00000080:00200008:1.0:1775764698.456007:0:60457:0:(file.c:6226:ll_getattr_dentry()) START file f21j.sanity-pcc:[0x200003ab3:0x2:0x0](ffff8b2390ec3248), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764698.456013:0:60457:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab3:0x2:0x0](ffff8b2390ec3248),name=f21j.sanity-pcc 00000002:00010000:1.0:1775764698.456054:0:60457:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab3:0x2:0x0]) in obj [0x200003ab3:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764698.456074:0:60457:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b9c5fa00/0x73cfd86cfc17b986 lrc: 2/1,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051041 expref: -99 pid: 60457 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764698.456094:0:60457:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b9c5fa00/0x73cfd86cfc17b986 lrc: 2/1,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051041 expref: -99 pid: 60457 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764698.456102:0:60457:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200003ab3:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764698.456108:0:60457:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab3:0x2:0x0](ffff8b2390ec3248) 00000080:00010000:1.0:1775764698.456111:0:60457:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab3:0x2:0x0](ffff8b2390ec3248) for lock 0x73cfd86cfc17b986 00000080:00010000:1.0:1775764698.456116:0:60457:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b986 from it ffffa60b0106fcd8 00010000:00010000:1.0:1775764698.456119:0:60457:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b9c5fa00/0x73cfd86cfc17b986 lrc: 3/1,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051041 expref: -99 pid: 60457 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764698.456126:0:60457:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b9c5fa00/0x73cfd86cfc17b986 lrc: 2/0,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051041 expref: -99 pid: 60457 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764698.456166:0:60457:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab3:0x2:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764698.456178:0:60457:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab3:0x2:0x0] 00000008:00010000:1.0:1775764698.456334:0:60457:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764698.456338:0:60457:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5c30, osc lock ffff8b2382cf7c80, flags 0x400001002 00010000:00010000:1.0:1775764698.456404:0:60457:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2388a49800 lock: 00000000a1b263f6/0x73cfd86cfc17b98d lrc: 3/1,0 mode: --/PR res: [0x280000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 60457 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764698.456414:0:60457:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8b2388a49800 lock: 00000000a1b263f6/0x73cfd86cfc17b98d lrc: 3/1,0 mode: --/PR res: [0x280000400:0x116:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 60457 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1775764698.457114:0:60457:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1775764698.463100:0:60457:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab3:0x2:0x0] updating i_size 0 i_blocks 1 00000020:00010000:1.0:1775764698.463109:0:60457:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f19b60 (ffff8b2387cba5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764698.463113:0:60457:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f19b60 (ffff8b2387cba5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764698.463120:0:60457:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5c30 (ffff8b2387cbacf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764698.463125:0:60457:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f19b60 (ffff8b2387cba5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764698.463128:0:60457:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5c30 (ffff8b2387cba5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764698.463183:0:60457:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab3:0x2:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1775764698.463589:0:60457:0:(file.c:2612:do_file_write_iter()) START file f21j.sanity-pcc:[0x200003ab3:0x2:0x0], ppos: 0, count: 5000 00000080:00200000:1.0:1775764698.463623:0:60457:0:(file.c:2058:ll_file_io_generic()) f21j.sanity-pcc: write ppos: 0, bytes: 5000 00000080:00200000:1.0:1775764698.463630:0:60457:0:(file.c:1815:ll_io_set_mirror()) f21j.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764698.463643:0:60457:0:(cl_io.c:214:cl_io_rw_init()) header@0000000043eaf8db[0x110, 1, [0x200003ab3:0x2:0x0] hash] 00000020:00200000:1.0:1775764698.463645:0:60457:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 5000) 0 0 00000080:00200000:1.0:1775764698.463650:0:60457:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab3:0x2:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764698.463670:0:60457:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 1] 00020000:00200000:1.0:1775764698.463681:0:60457:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 5000) 5000, 5000 00020000:00200000:1.0:1775764698.463685:0:60457:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764698.463713:0:60457:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 1] 00000008:00010000:1.0:1775764698.463873:0:60457:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764698.463878:0:60457:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5c30, osc lock ffff8b2382cf7c80, flags 0x0 00010000:00010000:1.0:1775764698.463900:0:60457:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b23ab6dec00 type 11 mode 2 res 10737419264/278 (0 8191) 00010000:00010000:1.0:1775764698.464377:0:60457:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2388a49800 lock: 000000007716e2ea/0x73cfd86cfc17b994 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x116:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 60457 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764698.464389:0:60457:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8b2388a49800 lock: 000000007716e2ea/0x73cfd86cfc17b994 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x116:0x0].0x0 rrc: 3 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 60457 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764698.464400:0:60457:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8b2388a49800 lock: 000000007716e2ea/0x73cfd86cfc17b994 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x116:0x0].0x0 rrc: 3 type: EXT [0->8191] (req 0->8191) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 60457 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764698.464425:0:60457:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job cp:d5685161-9db4-4c4c-ac8e-956c40da0bd9:60457:1862022098535552:192.168.202.128@tcp:101:cp.0 00000100:00100000:1.0:1775764698.466545:0:60457:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394fd80 going to sleep for 11 seconds 00000100:00100000:1.0:1775764698.469580:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200000:1.0:1775764699.214938:0:60485:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908),name=/ 00000002:00010000:1.0:1775764699.214986:0:60485:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764699.215013:0:60485:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740e00/0x73cfd86cfc17b955 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f92 expref: -99 pid: 60434 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764699.215033:0:60485:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740e00/0x73cfd86cfc17b955 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f92 expref: -99 pid: 60434 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:1.0:1775764699.215042:0:60485:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764699.215053:0:60485:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) 00000080:00010000:1.0:1775764699.215057:0:60485:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) for lock 0x73cfd86cfc17b955 00000080:00010000:1.0:1775764699.215066:0:60485:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b955 from it ffffa60b01c13bf8 00010000:00010000:1.0:1775764699.215070:0:60485:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740e00/0x73cfd86cfc17b955 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050f92 expref: -99 pid: 60434 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764699.215078:0:60485:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740e00/0x73cfd86cfc17b955 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f92 expref: -99 pid: 60434 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1775764699.215096:0:60485:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908), inode mode 41ed mask 1 00000080:00200000:1.0:1775764699.215104:0:60485:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d21j.sanity-pcc, flags=27 00000080:00200000:1.0:1775764699.215109:0:60485:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab3:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764699.215122:0:60485:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f21j.sanity-pcc, dir=[0x200003ab3:0x1:0x0](ffff8b238990cb08), file ffff8b2385294780, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764699.215131:0:60485:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f21j.sanity-pcc, dir=[0x200003ab3:0x1:0x0](ffff8b238990cb08), intent=open 00000002:00010000:1.0:1775764699.215338:0:60485:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: IXÞ J„Ïy³âQÔ\ö,[0x0:0x0:0x0]) in obj [0x200003ab3:0x1:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764699.215410:0:60485:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740800/0x73cfd86cfc17b9c5 lrc: 3/1,0 mode: --/CR res: [0x200003ab3:0x1:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.215419:0:60485:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740800/0x73cfd86cfc17b9c5 lrc: 3/1,0 mode: --/CR res: [0x200003ab3:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764699.215437:0:60485:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2388a49800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764699.215582:0:60485:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd180 x1862022098538880/t0(0) o101->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:12/10 lens 1072/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764699.215594:0:60485:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740800/0x73cfd86cfc17b9c5 lrc: 3/1,0 mode: --/CR res: [0x200003ab3:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764699.215625:0:60485:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lfs:d5685161-9db4-4c4c-ac8e-956c40da0bd9:60485:1862022098538880:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764699.217211:0:60485:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764699.223850:0:60485:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lfs:d5685161-9db4-4c4c-ac8e-956c40da0bd9:60485:1862022098538880:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764699.223886:0:60485:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740800/0x73cfd86cfc17b9c5 lrc: 4/1,0 mode: --/CR res: [0x200003ab3:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.223904:0:60485:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740800/0x73cfd86cfc17b9c5 lrc: 4/1,0 mode: --/CR res: [0x200003ab3:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.223913:0:60485:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740800/0x73cfd86cfc17b9c5 lrc: 4/1,0 mode: --/CR res: [0x200003ab3:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.223921:0:60485:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740800/0x73cfd86cfc17b9c5 lrc: 3/0,0 mode: --/CR res: [0x200003ab3:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.223930:0:60485:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740800/0x73cfd86cfc17b9c5 lrc: 4/0,0 mode: --/CR res: [0x200003ab3:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.223937:0:60485:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8740800 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764699.223943:0:60485:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740800/0x73cfd86cfc17b9c5 lrc: 4/0,0 mode: --/CR res: [0x200003ab3:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.223956:0:60485:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740800/0x73cfd86cfc17b9c5 lrc: 4/0,0 mode: --/CR res: [0x200003ab3:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.223965:0:60485:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740800/0x73cfd86cfc17b9c5 lrc: 5/0,0 mode: --/CR res: [0x200003ab3:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.223973:0:60485:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764699.223981:0:60485:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740800/0x73cfd86cfc17b9c5 lrc: 3/0,0 mode: --/CR res: [0x200003ab3:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.223987:0:60485:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740800/0x73cfd86cfc17b9c5 lrc: 0/0,0 mode: --/CR res: [0x200003ab3:0x1:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764699.224014:0:60485:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8b23864fd180 x1862022098538880/t0(0) o101->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:12/10 lens 1072/608 e 0 to 0 dl 1775764715 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764699.224033:0:60485:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab3:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1775764699.224256:0:60485:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908),name=/ 00000002:00010000:1.0:1775764699.224281:0:60485:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764699.224296:0:60485:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740e00/0x73cfd86cfc17b955 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f92 expref: -99 pid: 60434 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764699.224310:0:60485:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740e00/0x73cfd86cfc17b955 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f92 expref: -99 pid: 60434 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1775764699.224318:0:60485:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764699.224325:0:60485:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) 00000080:00010000:1.0:1775764699.224328:0:60485:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) for lock 0x73cfd86cfc17b955 00000080:00010000:1.0:1775764699.224334:0:60485:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b955 from it ffffa60b01c13bf8 00010000:00010000:1.0:1775764699.224340:0:60485:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740e00/0x73cfd86cfc17b955 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03050f92 expref: -99 pid: 60434 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764699.224347:0:60485:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8740e00/0x73cfd86cfc17b955 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03050f92 expref: -99 pid: 60434 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:1.0:1775764699.224365:0:60485:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908), inode mode 41ed mask 1 00000080:00200000:1.0:1775764699.224375:0:60485:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d21j.sanity-pcc, flags=267 00000080:00200000:1.0:1775764699.224384:0:60485:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab3:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764699.224396:0:60485:0:(file.c:1057:ll_file_open()) START file d21j.sanity-pcc:[0x200003ab3:0x1:0x0](ffff8b238990cb08), flags 304000 00000002:00010000:1.0:1775764699.224414:0:60485:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab3:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:1.0:1775764699.224952:0:60485:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8741c00/0x73cfd86cfc17b9cc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.224965:0:60485:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8741c00/0x73cfd86cfc17b9cc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764699.224982:0:60485:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2388a49800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764699.224992:0:60485:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd180 x1862022098539136/t0(0) o101->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764699.225004:0:60485:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8741c00/0x73cfd86cfc17b9cc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764699.225033:0:60485:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lfs:d5685161-9db4-4c4c-ac8e-956c40da0bd9:60485:1862022098539136:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764699.226209:0:60485:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764699.234752:0:60485:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lfs:d5685161-9db4-4c4c-ac8e-956c40da0bd9:60485:1862022098539136:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764699.234792:0:60485:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8741c00/0x73cfd86cfc17b9cc lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.234814:0:60485:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8741c00/0x73cfd86cfc17b9cc lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.234821:0:60485:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8741c00/0x73cfd86cfc17b9cc lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.234829:0:60485:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8741c00/0x73cfd86cfc17b9cc lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.234847:0:60485:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8741c00/0x73cfd86cfc17b9cc lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.234853:0:60485:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8741c00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764699.234858:0:60485:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8741c00/0x73cfd86cfc17b9cc lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.234866:0:60485:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8741c00/0x73cfd86cfc17b9cc lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.234874:0:60485:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8741c00/0x73cfd86cfc17b9cc lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.234882:0:60485:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764699.234890:0:60485:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8741c00/0x73cfd86cfc17b9cc lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.234896:0:60485:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b8741c00/0x73cfd86cfc17b9cc lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 60485 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764699.234918:0:60485:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23864fd180 x1862022098539136/t34359739004(34359739004) o101->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764715 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764699.234938:0:60485:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fd180 x1862022098539136/t34359739004(34359739004) o101->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764715 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764699.234954:0:60485:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764699.234976:0:60485:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab3:0x1:0x0], updating i_size 1024 00000002:00100000:1.0:1775764699.234996:0:60485:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fd180 x1862022098539136/t34359739004(34359739004) o101->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764715 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764699.235118:0:60485:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200003ab3:0x1:0x0](ffff8b238990cb08) cmd=c0086915 arg=1db63d0 00000100:00100000:1.0:1775764699.235229:0:60485:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job lfs:d5685161-9db4-4c4c-ac8e-956c40da0bd9:60485:1862022098539264:192.168.202.128@tcp:34:lfs.0 00000100:00100000:1.0:1775764699.235327:0:60485:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764699.239690:0:60485:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job lfs:d5685161-9db4-4c4c-ac8e-956c40da0bd9:60485:1862022098539264:192.168.202.128@tcp:34:lfs.0 00000080:00200008:1.0:1775764699.240117:0:60485:0:(file.c:447:ll_file_release()) START file d21j.sanity-pcc:[0x200003ab3:0x1:0x0](ffff8b238990cb08), flags 304000 00000002:00100000:1.0:1775764699.240186:0:60485:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fd180 x1862022098539136/t34359739004(34359739004) o101->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764715 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764699.240198:0:60485:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2388a49800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764699.240203:0:60485:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fc000 x1862022098539520/t0(0) o35->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764699.240228:0:60485:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job lfs:d5685161-9db4-4c4c-ac8e-956c40da0bd9:60485:1862022098539520:192.168.202.128@tcp:35:lfs.0 00000100:00100000:1.0:1775764699.240577:0:60485:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764699.243543:0:60485:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job lfs:d5685161-9db4-4c4c-ac8e-956c40da0bd9:60485:1862022098539520:192.168.202.128@tcp:35:lfs.0 00000002:00100000:1.0:1775764699.243577:0:60485:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fd180 x1862022098539136/t34359739004(34359739004) o101->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764715 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764699.321295:0:60491:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job cmp:d5685161-9db4-4c4c-ac8e-956c40da0bd9:60491:1862022098539776:192.168.202.128@tcp:101:cmp.0 00010000:00010000:1.0:1775764699.321326:0:60491:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044400/0x73cfd86cfc17b9d3 lrc: 4/1,0 mode: --/CR res: [0x200003ab3:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051136 expref: -99 pid: 60491 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.321345:0:60491:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044400/0x73cfd86cfc17b9d3 lrc: 4/1,0 mode: --/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051136 expref: -99 pid: 60491 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.321351:0:60491:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044400/0x73cfd86cfc17b9d3 lrc: 4/1,0 mode: --/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051136 expref: -99 pid: 60491 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.321362:0:60491:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044400/0x73cfd86cfc17b9d3 lrc: 4/1,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051136 expref: -99 pid: 60491 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.321370:0:60491:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044400/0x73cfd86cfc17b9d3 lrc: 4/1,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051136 expref: -99 pid: 60491 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.321384:0:60491:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044400/0x73cfd86cfc17b9d3 lrc: 5/2,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051136 expref: -99 pid: 60491 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.321392:0:60491:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044400/0x73cfd86cfc17b9d3 lrc: 5/2,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051136 expref: -99 pid: 60491 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.321400:0:60491:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044400/0x73cfd86cfc17b9d3 lrc: 4/1,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051136 expref: -99 pid: 60491 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764699.321412:0:60491:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b2382dace00 x1862022098539776/t34359739006(34359739006) o101->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:12/10 lens 1072/896 e 0 to 0 dl 1775764715 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cmp.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764699.321425:0:60491:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382dace00 x1862022098539776/t34359739006(34359739006) o101->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:12/10 lens 1072/896 e 0 to 0 dl 1775764715 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cmp.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764699.321440:0:60491:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044400/0x73cfd86cfc17b9d3 lrc: 3/1,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051136 expref: -99 pid: 60491 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764699.321907:0:60491:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044400/0x73cfd86cfc17b9d3 lrc: 3/1,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051136 expref: -99 pid: 60491 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.321923:0:60491:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b23893d5000 type 13 mode 4 res 8589949619/2 (0 0) 00000080:00200000:1.0:1775764699.321930:0:60491:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab3:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764699.322027:0:60491:0:(namei.c:147:ll_iget()) got inode: [0x200003ab3:0x2:0x0](ffff8b2390ec3248): rc = 0 00000080:00200000:1.0:1775764699.322312:0:60491:0:(file.c:6766:ll_layout_conf()) [0x200003ab3:0x2:0x0]: layout version change: 2 -> 2 00000080:00010000:1.0:1775764699.322336:0:60491:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab3:0x2:0x0](ffff8b2390ec3248) for lock 0x73cfd86cfc17b9d3 00000080:00010000:1.0:1775764699.322354:0:60491:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab3:0x2:0x0](ffff8b2390ec3248) 00000080:00200008:1.0:1775764699.322375:0:60491:0:(file.c:1057:ll_file_open()) START file f21j.sanity-pcc:[0x200003ab3:0x2:0x0](ffff8b2390ec3248), flags 100000 00000002:00100000:1.0:1775764699.322408:0:60491:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382dace00 x1862022098539776/t34359739006(34359739006) o101->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:12/10 lens 792/896 e 0 to 0 dl 1775764715 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cmp.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1775764699.322483:0:60491:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b9d3 from it ffff8b23891e85a0 00010000:00010000:1.0:1775764699.322488:0:60491:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044400/0x73cfd86cfc17b9d3 lrc: 3/1,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051136 expref: -99 pid: 60491 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764699.322500:0:60491:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044400/0x73cfd86cfc17b9d3 lrc: 2/0,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051136 expref: -99 pid: 60491 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764699.322539:0:60491:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab3:0x2:0x0](ffff8b2390ec3248), inode mode 81a4 mask 44 00000080:00200008:1.0:1775764699.322581:0:60491:0:(file.c:6226:ll_getattr_dentry()) START file f21j.sanity-pcc:[0x200003ab3:0x2:0x0](ffff8b2390ec3248), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764699.322587:0:60491:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab3:0x2:0x0](ffff8b2390ec3248),name=f21j.sanity-pcc 00000002:00010000:1.0:1775764699.322633:0:60491:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab3:0x2:0x0]) in obj [0x200003ab3:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764699.322646:0:60491:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044400/0x73cfd86cfc17b9d3 lrc: 2/1,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051136 expref: -99 pid: 60491 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764699.322658:0:60491:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044400/0x73cfd86cfc17b9d3 lrc: 2/1,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051136 expref: -99 pid: 60491 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764699.322665:0:60491:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200003ab3:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764699.322671:0:60491:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab3:0x2:0x0](ffff8b2390ec3248) 00000080:00010000:1.0:1775764699.322675:0:60491:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab3:0x2:0x0](ffff8b2390ec3248) for lock 0x73cfd86cfc17b9d3 00000080:00010000:1.0:1775764699.322678:0:60491:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17b9d3 from it ffffa60b01d6bcd8 00010000:00010000:1.0:1775764699.322681:0:60491:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044400/0x73cfd86cfc17b9d3 lrc: 3/1,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051136 expref: -99 pid: 60491 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764699.322689:0:60491:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044400/0x73cfd86cfc17b9d3 lrc: 2/0,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051136 expref: -99 pid: 60491 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:1.0:1775764699.322862:0:60491:0:(file.c:5170:ll_file_seek()) START file f21j.sanity-pcc:[0x200003ab3:0x2:0x0], offset: 0, type: SEEK_CUR 00000080:00200008:1.0:1775764699.322870:0:60491:0:(file.c:5201:ll_file_seek()) COMPLETED file f21j.sanity-pcc:[0x200003ab3:0x2:0x0], offset: 0, type: SEEK_CUR, rc = 0 00000080:00200008:1.0:1775764699.322911:0:60491:0:(file.c:2454:do_file_read_iter()) START file f21j.sanity-pcc:[0x200003ab3:0x2:0x0], ppos: 0, count: 4194304 00000100:00100000:1.0:1775764699.719690:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d5685161-9db4-4c4c-ac8e-956c40da0bd9:2376:1862022098540416:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764700.494671:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119035520 00000100:00100000:1.0:1775764700.494683:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119035520 00000100:00100000:1.0:1775764700.494757:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 199 00000100:00100000:1.0:1775764700.494768:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daf480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43212:x1862022119035520:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764700.494786:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b2388a49800 lock: 000000001a0fdb8f/0x73cfd86cfc17b9f6 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x117:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e030511a6 expref: -99 pid: 60553 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764700.494888:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764700.494895:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daf480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43212:x1862022119035520:12345-192.168.202.128@tcp:104: Request processed in 129us (358us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764700.494902:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 199 00000100:00100000:1.0:1775764700.502070:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764700.746820:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d5685161-9db4-4c4c-ac8e-956c40da0bd9:2376:1862022098543744:192.168.202.128@tcp:3:lfs.0 00000008:00010000:1.0:1775764700.750243:0:2376:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8b2384e75480 00000020:00200000:1.0:1775764700.750266:0:2376:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1775764700.750289:0:2376:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8b2384e75480 00000100:00100000:1.0:1775764700.750320:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:d5685161-9db4-4c4c-ac8e-956c40da0bd9:2376:1862022098543744:192.168.202.128@tcp:3:lfs.0 00000020:00010000:1.0:1775764701.171526:0:60593:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384cd6d58 (ffff8b238a3ffd88/0) at cl_lock_release():245 00000020:00010000:1.0:1775764701.171532:0:60593:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384cd6d58 (ffff8b238a3ffd88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764701.171536:0:60593:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23b8563e30 (ffff8b238a3ff1b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764701.171547:0:60593:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2388a49800 lock: 0000000026ad166b/0x73cfd86cfc17b9fd lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030511ad expref: -99 pid: 60553 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764701.171562:0:60593:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2388a49800 lock: 0000000026ad166b/0x73cfd86cfc17b9fd lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030511ad expref: -99 pid: 60553 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00010000:1.0:1775764701.171578:0:60593:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384cd6d58 (ffff8b238a3ffd88/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764701.171581:0:60593:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23b8563e30 (ffff8b238a3ffd88/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764701.171590:0:60593:0:(vvp_io.c:517:vvp_io_advance()) advancing 5000 bytes 00000080:00200000:1.0:1775764701.171620:0:60593:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab3:0x3:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764701.171625:0:60593:0:(file.c:2206:ll_file_io_generic()) f21j.sanity-pcc_2: 1 io complete with rc: 0, result: 5000, restart: 0 00000080:00200000:1.0:1775764701.171632:0:60593:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 5000 00000080:00200008:1.0:1775764701.171673:0:60593:0:(file.c:2454:do_file_read_iter()) START file f21j.sanity-pcc_2:[0x200003ab3:0x3:0x0], ppos: 5000, count: 4189304 00000080:00200000:1.0:1775764701.171707:0:60593:0:(file.c:2058:ll_file_io_generic()) f21j.sanity-pcc_2: read ppos: 5000, bytes: 4189304 00000080:00200000:1.0:1775764701.171714:0:60593:0:(file.c:1815:ll_io_set_mirror()) f21j.sanity-pcc_2: desiginated mirror: 0 00000020:00200000:1.0:1775764701.171721:0:60593:0:(cl_io.c:214:cl_io_rw_init()) header@0000000083af2673[0x110, 3, [0x200003ab3:0x3:0x0] hash] 00000020:00200000:1.0:1775764701.171723:0:60593:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [5000, 4194304) 0 0 00000080:00200000:1.0:1775764701.171728:0:60593:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab3:0x3:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764701.171746:0:60593:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab3:0x3:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764701.171749:0:60593:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:1.0:1775764701.171757:0:60593:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 5000 chunk: [5000, 4194304) 4194304, 4189304 00020000:00200000:1.0:1775764701.171761:0:60593:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764701.171790:0:60593:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [1, 1023] 00000008:00010000:1.0:1775764701.171806:0:60593:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764701.171811:0:60593:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c52b0, osc lock ffff8b2382cf7300, flags 0x400000000000000 00010000:00010000:1.0:1775764701.171845:0:60593:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2388a49800 lock: 0000000026ad166b/0x73cfd86cfc17b9fd lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x117:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030511ad expref: -99 pid: 60553 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764701.171861:0:60593:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (4096 4194303) ns: lustre-OST0000-osc-ffff8b2388a49800 lock: 0000000026ad166b/0x73cfd86cfc17b9fd lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030511ad expref: -99 pid: 60553 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764701.171873:0:60593:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2388a49800 lock: 0000000026ad166b/0x73cfd86cfc17b9fd lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030511ad expref: -99 pid: 60553 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764701.171882:0:60593:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2388a49800 lock: 0000000026ad166b/0x73cfd86cfc17b9fd lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030511ad expref: -99 pid: 60553 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764701.171890:0:60593:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2388a49800 lock: 0000000026ad166b/0x73cfd86cfc17b9fd lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030511ad expref: -99 pid: 60553 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000080:00200000:1.0:1775764701.171905:0:60593:0:(vvp_io.c:826:vvp_io_read_start()) f21j.sanity-pcc_2: read [5000, 4194304) 00000001:00010000:1.0:1775764701.171920:0:60593:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab3:0x3:0x0] 00000008:00010000:1.0:1775764701.171926:0:60593:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764701.171928:0:60593:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5430, osc lock ffff8b2382cf6850, flags 0x400000400001002 00010000:00010000:1.0:1775764701.171936:0:60593:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2388a49800 lock: 0000000026ad166b/0x73cfd86cfc17b9fd lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x117:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030511ad expref: -99 pid: 60553 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764701.171945:0:60593:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2388a49800 lock: 0000000026ad166b/0x73cfd86cfc17b9fd lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030511ad expref: -99 pid: 60553 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00200000:1.0:1775764701.171953:0:60593:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764701.171956:0:60593:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2388a49800 lock: 0000000026ad166b/0x73cfd86cfc17b9fd lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030511ad expref: -99 pid: 60553 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764701.171963:0:60593:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2388a49800 lock: 0000000026ad166b/0x73cfd86cfc17b9fd lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030511ad expref: -99 pid: 60553 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00200000:1.0:1775764701.171969:0:60593:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764701.171977:0:60593:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab3:0x3:0x0] updating i_size 5000 i_blocks 1 00000020:00010000:1.0:1775764701.171981:0:60593:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f196d0 (ffff8b2387cba908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764701.171983:0:60593:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f196d0 (ffff8b2387cba908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764701.171985:0:60593:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5430 (ffff8b2387cba248/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764701.171987:0:60593:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f196d0 (ffff8b2387cba908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764701.171989:0:60593:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5430 (ffff8b2387cba908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764701.171995:0:60593:0:(vvp_io.c:885:vvp_io_read_start()) count:4189304 iocb pos:5000 00000020:00010000:1.0:1775764701.172001:0:60593:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb7128 (ffff8b2387cba908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764701.172004:0:60593:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb7128 (ffff8b2387cba908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764701.172005:0:60593:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c52b0 (ffff8b2387cba248/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764701.172009:0:60593:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2388a49800 lock: 0000000026ad166b/0x73cfd86cfc17b9fd lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030511ad expref: -99 pid: 60553 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764701.172016:0:60593:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2388a49800 lock: 0000000026ad166b/0x73cfd86cfc17b9fd lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030511ad expref: -99 pid: 60553 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000020:00010000:1.0:1775764701.172023:0:60593:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb7128 (ffff8b2387cba908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764701.172025:0:60593:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c52b0 (ffff8b2387cba908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764701.172028:0:60593:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:1.0:1775764701.172047:0:60593:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab3:0x3:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764701.172050:0:60593:0:(file.c:2206:ll_file_io_generic()) f21j.sanity-pcc_2: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:1.0:1775764701.172053:0:60593:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:1.0:1775764701.172125:0:60593:0:(file.c:447:ll_file_release()) START file f21j.sanity-pcc_2:[0x200003ab3:0x3:0x0](ffff8b2390ec5348), flags 100000 00010000:00010000:1.0:1775764701.172149:0:60593:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b9c5e200/0x73cfd86cfc17ba43 lrc: 2/0,0 mode: CR/CR res: [0x200003ab3:0x3:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030512a9 expref: -99 pid: 60593 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000100:00100000:1.0:1775764701.201025:0:60594:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job rm:d5685161-9db4-4c4c-ac8e-956c40da0bd9:60594:1862022098546176:192.168.202.128@tcp:35:rm.0 00000002:00100000:1.0:1775764701.201069:0:60594:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864f0e00 x1862022098545792/t34359739021(34359739021) o101->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764717 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cmp.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764701.201259:0:60594:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab3:0x3:0x0]: losing layout lock 00010000:00010000:1.0:1775764701.201321:0:60594:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eac00/0x73cfd86cfc17ba20 lrc: 2/0,0 mode: CR/CR res: [0x200003ab3:0x3:0x0].0x0 bits 0xa/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03051216 expref: -99 pid: 60565 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764701.201347:0:60594:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab3:0x3:0x0]: losing layout lock 00010000:00010000:1.0:1775764701.201360:0:60594:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea400/0x73cfd86cfc17ba35 lrc: 2/0,0 mode: PR/PR res: [0x200003ab3:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03051286 expref: -99 pid: 60587 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764701.201377:0:60594:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab3:0x3:0x0]: losing layout lock 00010000:00010000:1.0:1775764701.201389:0:60594:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb600/0x73cfd86cfc17ba04 lrc: 2/0,0 mode: PR/PR res: [0x200003ab3:0x3:0x0].0x0 bits 0x20/0x0 rrc: 6 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e030511bb expref: -99 pid: 60553 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764701.201628:0:60594:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b9c5e200/0x73cfd86cfc17ba43 lrc: 1/0,0 mode: --/CR res: [0x200003ab3:0x3:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030512a9 expref: -99 pid: 60593 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764701.201637:0:60594:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eac00/0x73cfd86cfc17ba20 lrc: 1/0,0 mode: --/CR res: [0x200003ab3:0x3:0x0].0x0 bits 0xa/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051216 expref: -99 pid: 60565 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764701.201643:0:60594:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea400/0x73cfd86cfc17ba35 lrc: 1/0,0 mode: --/PR res: [0x200003ab3:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051286 expref: -99 pid: 60587 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764701.201648:0:60594:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb600/0x73cfd86cfc17ba04 lrc: 1/0,0 mode: --/PR res: [0x200003ab3:0x3:0x0].0x0 bits 0x20/0x0 rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030511bb expref: -99 pid: 60553 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764701.201656:0:60594:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044200/0x73cfd86cfc17ba12 lrc: 1/0,0 mode: --/PR res: [0x200003ab3:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030511ec expref: -99 pid: 60565 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764701.201663:0:60594:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 5 locks packed 00010000:00010000:1.0:1775764701.201666:0:60594:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b9c5e200/0x73cfd86cfc17ba43 lrc: 0/0,0 mode: --/CR res: [0x200003ab3:0x3:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030512a9 expref: -99 pid: 60593 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764701.201684:0:60594:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eac00/0x73cfd86cfc17ba20 lrc: 0/0,0 mode: --/CR res: [0x200003ab3:0x3:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051216 expref: -99 pid: 60565 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764701.201693:0:60594:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6ea400/0x73cfd86cfc17ba35 lrc: 0/0,0 mode: --/PR res: [0x200003ab3:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051286 expref: -99 pid: 60587 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764701.201871:0:60594:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb600/0x73cfd86cfc17ba04 lrc: 0/0,0 mode: --/PR res: [0x200003ab3:0x3:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030511bb expref: -99 pid: 60553 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764701.201887:0:60594:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23c2044200/0x73cfd86cfc17ba12 lrc: 0/0,0 mode: --/PR res: [0x200003ab3:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030511ec expref: -99 pid: 60565 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000020:00100000:1.0:1775764701.201912:0:60594:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2388a49800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1775764701.201922:0:60594:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd180 x1862022098546432/t0(0) o36->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:12/10 lens 528/1032 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764702.559535:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119036544 00000100:00100000:1.0:1775764702.559546:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119036544 00000100:00100000:1.0:1775764702.559607:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 201 00000100:00100000:1.0:1775764702.559621:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dace00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:d5685161-9db4-4c4c-ac8e-956c40da0bd9+5:44493:x1862022119036544:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764702.559644:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23b9c5f000/0x73cfd86cfc17ba5f lrc: 2/0,0 mode: PR/PR res: [0x200003ab3:0x2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03051327 expref: -99 pid: 60634 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764702.559747:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764702.559754:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dace00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:d5685161-9db4-4c4c-ac8e-956c40da0bd9+5:44493:x1862022119036544:12345-192.168.202.128@tcp:104: Request processed in 136us (498us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764702.559763:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 201 00000100:00100000:1.0:1775764707.396098:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098549120:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764707.396114:0:2377:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b2388a49800: committing for last_committed 34359738432 gen 1 00000100:00100000:1.0:1775764707.396140:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d5685161-9db4-4c4c-ac8e-956c40da0bd9:2377:1862022098549376:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764707.396153:0:2377:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2388a49800: committing for last_committed 34359739027 gen 1 00000100:00100000:1.0:1775764707.396179:0:2377:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23b9ea7100 x1862022098548096/t34359739026(34359739026) o101->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764718 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'find.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764707.396190:0:2377:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23b9ea5f80 x1862022098548736/t34359739027(34359739027) o101->lustre-MDT0000-mdc-ffff8b2388a49800@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764718 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764707.396198:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d5685161-9db4-4c4c-ac8e-956c40da0bd9:2377:1862022098549248:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764707.396208:0:2377:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b2388a49800: committing for last_committed 34359738437 gen 1 00000100:00100000:1.0:1775764707.396221:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:d5685161-9db4-4c4c-ac8e-956c40da0bd9:2377:1862022098549504:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764707.398149:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098548992:192.168.202.128@tcp:400:kworker.0 00000080:00200000:1.0:1775764711.113835:0:61080:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908),name=/ 00000002:00010000:1.0:1775764711.113867:0:61080:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764711.113893:0:61080:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb000/0x73cfd86cfc17ba4a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512da expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1775764711.113912:0:61080:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb000/0x73cfd86cfc17ba4a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512da expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:1.0:1775764711.113922:0:61080:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764711.113932:0:61080:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) 00000080:00010000:1.0:1775764711.113936:0:61080:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) for lock 0x73cfd86cfc17ba4a 00000080:00010000:1.0:1775764711.113944:0:61080:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17ba4a from it ffffa60b01dcbb58 00010000:00010000:1.0:1775764711.113948:0:61080:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb000/0x73cfd86cfc17ba4a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030512da expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1775764711.113958:0:61080:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb000/0x73cfd86cfc17ba4a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512da expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1775764711.113982:0:61080:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908), inode mode 41ed mask 1 00000080:00200000:1.0:1775764711.113990:0:61080:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d21j.sanity-pcc, flags=8 00000080:00200008:1.0:1775764711.114001:0:61080:0:(file.c:6226:ll_getattr_dentry()) START file d21j.sanity-pcc:[0x200003ab3:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764711.114007:0:61080:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab3:0x1:0x0](ffff8b238990cb08),name=d21j.sanity-pcc 00000002:00010000:1.0:1775764711.114020:0:61080:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab3:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764711.114029:0:61080:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb400/0x73cfd86cfc17ba51 lrc: 2/1,0 mode: PR/PR res: [0x200003ab3:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512f6 expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764711.114038:0:61080:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb400/0x73cfd86cfc17ba51 lrc: 2/1,0 mode: PR/PR res: [0x200003ab3:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512f6 expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200000:1.0:1775764711.114045:0:61080:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764711.114050:0:61080:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab3:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764711.114052:0:61080:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab3:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17ba51 00000080:00010000:1.0:1775764711.114055:0:61080:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17ba51 from it ffffa60b01dcbcb8 00010000:00010000:1.0:1775764711.114058:0:61080:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb400/0x73cfd86cfc17ba51 lrc: 3/1,0 mode: PR/PR res: [0x200003ab3:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030512f6 expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764711.114066:0:61080:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb400/0x73cfd86cfc17ba51 lrc: 2/0,0 mode: PR/PR res: [0x200003ab3:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512f6 expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200000:1.0:1775764711.114095:0:61080:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908),name=/ 00000002:00010000:1.0:1775764711.114102:0:61080:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764711.114108:0:61080:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb000/0x73cfd86cfc17ba4a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512da expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764711.114211:0:61080:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb000/0x73cfd86cfc17ba4a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512da expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1775764711.114219:0:61080:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764711.114223:0:61080:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) 00000080:00010000:1.0:1775764711.114226:0:61080:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) for lock 0x73cfd86cfc17ba4a 00000080:00010000:1.0:1775764711.114230:0:61080:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17ba4a from it ffffa60b01dcbbf8 00010000:00010000:1.0:1775764711.114234:0:61080:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb000/0x73cfd86cfc17ba4a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030512da expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764711.114242:0:61080:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb000/0x73cfd86cfc17ba4a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512da expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764711.114251:0:61080:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908), inode mode 41ed mask 1 00000080:00200000:1.0:1775764711.114255:0:61080:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d21j.sanity-pcc, flags=266 00000080:00200000:1.0:1775764711.114260:0:61080:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab3:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764711.114267:0:61080:0:(file.c:1057:ll_file_open()) START file d21j.sanity-pcc:[0x200003ab3:0x1:0x0](ffff8b238990cb08), flags 704400 00000080:00200008:1.0:1775764711.114286:0:61080:0:(file.c:6226:ll_getattr_dentry()) START file d21j.sanity-pcc:[0x200003ab3:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764711.114291:0:61080:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab3:0x1:0x0](ffff8b238990cb08),name=d21j.sanity-pcc 00000002:00010000:1.0:1775764711.114298:0:61080:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab3:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764711.114305:0:61080:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb400/0x73cfd86cfc17ba51 lrc: 2/1,0 mode: PR/PR res: [0x200003ab3:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512f6 expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764711.114362:0:61080:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb400/0x73cfd86cfc17ba51 lrc: 2/1,0 mode: PR/PR res: [0x200003ab3:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512f6 expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200000:1.0:1775764711.114375:0:61080:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764711.114379:0:61080:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab3:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764711.114381:0:61080:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab3:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17ba51 00000080:00010000:1.0:1775764711.114385:0:61080:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17ba51 from it ffffa60b01dcbcd8 00010000:00010000:1.0:1775764711.114389:0:61080:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb400/0x73cfd86cfc17ba51 lrc: 3/1,0 mode: PR/PR res: [0x200003ab3:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030512f6 expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764711.114395:0:61080:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb400/0x73cfd86cfc17ba51 lrc: 2/0,0 mode: PR/PR res: [0x200003ab3:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512f6 expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:1.0:1775764711.114438:0:61080:0:(dir.c:331:ll_iterate()) VFS Op:inode=[0x200003ab3:0x1:0x0](ffff8b238990cb08) pos/size0/7168 32bit_api 0 00000080:00200008:1.0:1775764711.114714:0:61080:0:(file.c:447:ll_file_release()) START file d21j.sanity-pcc:[0x200003ab3:0x1:0x0](ffff8b238990cb08), flags 704000 00010000:00010000:1.0:1775764711.114741:0:61080:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb800/0x73cfd86cfc17ba58 lrc: 2/0,0 mode: CR/CR res: [0x200003ab3:0x1:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051304 expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764711.114770:0:61080:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908),name=/ 00000002:00010000:1.0:1775764711.114776:0:61080:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764711.114782:0:61080:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb000/0x73cfd86cfc17ba4a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512da expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764711.114789:0:61080:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb000/0x73cfd86cfc17ba4a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512da expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764711.114795:0:61080:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764711.114799:0:61080:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) 00000080:00010000:1.0:1775764711.114801:0:61080:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2390ec0908) for lock 0x73cfd86cfc17ba4a 00000080:00010000:1.0:1775764711.114805:0:61080:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17ba4a from it ffffa60b01dcbbf8 00010000:00010000:1.0:1775764711.114808:0:61080:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb000/0x73cfd86cfc17ba4a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030512da expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764711.114815:0:61080:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb000/0x73cfd86cfc17ba4a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512da expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1775764711.114823:0:61080:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2390ec0908), inode mode 41ed mask 1 00000080:00200000:1.0:1775764711.114828:0:61080:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d21j.sanity-pcc, flags=266 00000080:00200000:1.0:1775764711.114832:0:61080:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab3:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764711.114836:0:61080:0:(file.c:1057:ll_file_open()) START file d21j.sanity-pcc:[0x200003ab3:0x1:0x0](ffff8b238990cb08), flags 704400 00000080:00200008:1.0:1775764711.114846:0:61080:0:(file.c:6226:ll_getattr_dentry()) START file d21j.sanity-pcc:[0x200003ab3:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764711.114850:0:61080:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab3:0x1:0x0](ffff8b238990cb08),name=d21j.sanity-pcc 00000002:00010000:1.0:1775764711.114857:0:61080:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab3:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764711.114863:0:61080:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb400/0x73cfd86cfc17ba51 lrc: 2/1,0 mode: PR/PR res: [0x200003ab3:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512f6 expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764711.114870:0:61080:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb400/0x73cfd86cfc17ba51 lrc: 2/1,0 mode: PR/PR res: [0x200003ab3:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512f6 expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:1.0:1775764711.114876:0:61080:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764711.114880:0:61080:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab3:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764711.114882:0:61080:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab3:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17ba51 00000080:00010000:1.0:1775764711.114886:0:61080:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17ba51 from it ffffa60b01dcbcd8 00010000:00010000:1.0:1775764711.114888:0:61080:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb400/0x73cfd86cfc17ba51 lrc: 3/1,0 mode: PR/PR res: [0x200003ab3:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030512f6 expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764711.114894:0:61080:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2388a49800 lock: ffff8b23ac6eb400/0x73cfd86cfc17ba51 lrc: 2/0,0 mode: PR/PR res: [0x200003ab3:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030512f6 expref: -99 pid: 60628 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:1.0:1775764711.114944:0:61080:0:(llite_lib.c:2726:ll_statfs()) VFS Op:sb=lustre (ffff8b2388a49800) d21j.sanity-pcc proj=0 00000080:00000004:1.0:1775764711.114949:0:61080:0:(obd_class.h:1092:obd_statfs()) lustre-clilmv-ffff8b2388a49800: age 2168, max_age 2176 00800000:00000004:1.0:1775764711.114958:0:61080:0:(obd_class.h:1092:obd_statfs()) lustre-MDT0000-mdc-ffff8b2388a49800: age 2168, max_age 2176 00000100:00100000:1.0:1775764711.115019:0:61080:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job rm:d5685161-9db4-4c4c-ac8e-956c40da0bd9:61080:1862022098550144:192.168.202.128@tcp:41:rm.0 00000100:00100000:1.0:1775764711.128078:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764711.203018:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200008:1.0:1775764711.763687:0:61090:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764711.763706:0:61090:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:1.0:1775764711.763750:0:61090:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764711.763844:0:61090:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741a00/0x73cfd86cfc17ba82 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 61090 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764711.763858:0:61090:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741a00/0x73cfd86cfc17ba82 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61090 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764711.763870:0:61090:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741a00/0x73cfd86cfc17ba82 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61090 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764711.763906:0:61090:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job lsof:82781e18-e0e3-4f9f-9a2b-041bf0344414:61090:1862022098551808:192.168.202.128@tcp:101:lsof.0 00000100:00100000:1.0:1775764711.764311:0:61090:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2382470e40 going to sleep for 11 seconds 00010000:00010000:1.0:1775764713.743142:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000080:1.0:1775764713.753363:0:36:0:(genops.c:863:class_export_destroy()) destroying export ffff8b238a0af000/LOV_OSC_UUID for lustre-OST0001-osc-ffff8b2388a49800 00000020:00000080:1.0:1775764713.754336:0:36:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff8b23830a7000 for lustre-OST0001-osc-ffff8b2388a49800 00000020:00000080:1.0:1775764713.794450:0:36:0:(genops.c:863:class_export_destroy()) destroying export ffff8b238a0a8000/LOV_OSC_UUID for lustre-OST0000-osc-ffff8b2388a49800 00000020:00000080:1.0:1775764713.836831:0:36:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff8b23830a7800 for lustre-MDT0000-mdc-ffff8b2388a49800 00000020:00000080:1.0:1775764713.836852:0:36:0:(genops.c:904:class_export_put()) final put ffff8b23830a4000/d5685161-9db4-4c4c-ac8e-956c40da0bd9 00000020:00000080:1.0:1775764713.836857:0:36:0:(genops.c:863:class_export_destroy()) destroying export ffff8b23830a4000/d5685161-9db4-4c4c-ac8e-956c40da0bd9 for lustre-MDT0000-mdc-ffff8b2388a49800 00000020:01000000:1.0:1775764713.836871:0:36:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-MDT0000-mdc-ffff8b2388a49800 (d5685161-9db4-4c4c-ac8e-956c40da0bd9) 00000100:00100000:1.0:1775764714.120123:0:61108:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job mount.lustre:24b54882-c0a1-43f2-954f-009e5a9afc9d:61108:1862022098553600:192.168.202.128@tcp:101:mount.lustre.0 00010000:00010000:1.0:1775764714.120157:0:61108:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.202.128@tcp lock: ffff8b23b9c5f600/0x73cfd86cfc17ba90 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x11000000000000 nid: local remote: 0x57b8ff0e030513dd expref: -99 pid: 61108 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764714.120175:0:61108:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.202.128@tcp lock: ffff8b23b9c5f600/0x73cfd86cfc17ba90 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1000000000000 nid: local remote: 0x57b8ff0e030513dd expref: -99 pid: 61108 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764714.120269:0:61108:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job mount.lustre:24b54882-c0a1-43f2-954f-009e5a9afc9d:61108:1862022098553728:192.168.202.128@tcp:501:mount.lustre.0 00000100:00100000:1.0:1775764714.120476:0:61108:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2383f4fcc0 going to sleep for 11 seconds 00000020:01000000:1.0:1775764714.231445:0:61108:0:(obd_config.c:2143:class_config_parse_llog()) Processed log lustre-client gen 1-41 (rc=0) 10000000:01000000:1.0:1775764714.231509:0:61108:0:(mgc_request.c:1896:mgc_process_log()) MGC192.168.202.128@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:1.0:1775764714.231534:0:61108:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.202.128@tcp lock: ffff8b23b9c5f600/0x73cfd86cfc17ba90 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x11000000000000 nid: local remote: 0x57b8ff0e030513dd expref: -99 pid: 61108 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764714.231552:0:61108:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.202.128@tcp lock: ffff8b23b9c5f600/0x73cfd86cfc17ba90 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x11000000000000 nid: local remote: 0x57b8ff0e030513dd expref: -99 pid: 61108 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1775764714.231562:0:61108:0:(mgc_request.c:1819:mgc_process_log()) Process log lustre-cliir-ffff8b2389398000 from 1 10000000:01000000:1.0:1775764714.231566:0:61108:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:1.0:1775764714.231631:0:61108:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.202.128@tcp lock: ffff8b23b8740600/0x73cfd86cfc17baba lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 61108 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764714.231641:0:61108:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.202.128@tcp lock: ffff8b23b8740600/0x73cfd86cfc17baba lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 61108 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764714.231656:0:61108:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.202.128@tcp lock: ffff8b23b8740600/0x73cfd86cfc17baba lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 61108 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764714.231691:0:61108:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job mount.lustre:24b54882-c0a1-43f2-954f-009e5a9afc9d:61108:1862022098554240:192.168.202.128@tcp:101:mount.lustre.0 00000100:00100000:1.0:1775764714.231798:0:61108:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85e13c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764714.409554:0:61122:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:24b54882-c0a1-43f2-954f-009e5a9afc9d:61122:1862022098555008:192.168.202.128@tcp:502:llog_process_th.0 00000020:01000000:1.0:1775764714.409628:0:61122:0:(obd_config.c:1888:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1775764714.409639:0:61122:0:(obd_config.c:1468:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1775764714.409642:0:61122:0:(obd_config.c:1559:class_process_config()) marker 4 (0x2) lustre mdt.lustre-MDT0000.hsm_control 00000040:00080000:1.0:1775764714.409649:0:61122:0:(llog.c:857:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 10 count 10 00000100:00100000:1.0:1775764714.436909:0:2375:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 0 gen 1 00000100:00080000:1.0:1775764714.436929:0:2375:0:(import.c:1130:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffff8b2389398000: connect to target with instance 8 00000100:00080000:1.0:1775764714.436939:0:2375:0:(import.c:958:ptlrpc_connect_set_flags()) lustre-MDT0000-mdc-ffff8b2389398000: Resetting ns_connect_flags to server flags: 0xae7a5e7be344d3b8 00000100:00080000:1.0:1775764714.436951:0:2375:0:(import.c:1211:ptlrpc_connect_interpret()) connected to replayable target: lustre-MDT0000_UUID 00000100:00080000:1.0:1775764714.436955:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to FULL 00000100:00080000:1.0:1775764714.437096:0:2375:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 0000000057935a61): req@ffff8b23864f2300 x1862022098555392/t0(0) o41->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764714.437115:0:2375:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 00000000ae427fbe): req@ffff8b23864f1c00 x1862022098555264/t0(0) o41->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764714.437125:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098555136:192.168.202.128@tcp:38:mount.lustre.0 00000100:00100000:1.0:1775764714.437235:0:61108:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f600 going to sleep for 11 seconds 00000100:00100000:1.0:1775764714.469436:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098555776:192.168.202.128@tcp:8:mount.lustre.0 00000100:00100000:1.0:1775764714.469514:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098555904:192.168.202.128@tcp:8:mount.lustre.0 00000100:00100000:1.0:1775764714.476830:0:2375:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b2389398000: committing for last_committed 0 gen 1 00000100:00080000:1.0:1775764714.476850:0:2375:0:(import.c:1130:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffff8b2389398000: connect to target with instance 8 00000100:00080000:1.0:1775764714.476862:0:2375:0:(import.c:958:ptlrpc_connect_set_flags()) lustre-OST0000-osc-ffff8b2389398000: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:1.0:1775764714.476876:0:2375:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0465af2e3640038 -> 0xa0425af2e3440038 00000080:00080000:1.0:1775764714.476882:0:2375:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:1.0:1775764714.476891:0:2375:0:(import.c:1211:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0000_UUID 00000100:00080000:1.0:1775764714.476896:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to FULL 00020000:01000000:1.0:1775764714.476905:0:2375:0:(lov_obd.c:392:lov_set_osc_active()) Marking OSC lustre-OST0000_UUID active 00000080:00000004:1.0:1775764714.476907:0:2375:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:1.0:1775764714.476909:0:2375:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:1.0:1775764714.476920:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098555776:192.168.202.128@tcp:8:mount.lustre.0 00000100:00100000:1.0:1775764714.476931:0:2375:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b2389398000: committing for last_committed 0 gen 1 00000100:00080000:1.0:1775764714.476935:0:2375:0:(import.c:1130:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffff8b2389398000: connect to target with instance 8 00000100:00080000:1.0:1775764714.476939:0:2375:0:(import.c:958:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffff8b2389398000: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:1.0:1775764714.476943:0:2375:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:1.0:1775764714.476945:0:2375:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:1.0:1775764714.476947:0:2375:0:(import.c:1211:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:1.0:1775764714.476949:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00020000:01000000:1.0:1775764714.476952:0:2375:0:(lov_obd.c:392:lov_set_osc_active()) Marking OSC lustre-OST0001_UUID active 00000080:00000004:1.0:1775764714.476954:0:2375:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:1.0:1775764714.476956:0:2375:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:1.0:1775764714.476960:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098555904:192.168.202.128@tcp:8:mount.lustre.0 00010000:00010000:1.0:1775764718.524097:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1775764719.219060:0:61306:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-pcc test 22: Test RO-PCC attach for the HSM released file ========================================================== 15:58:37 (1775764717) 00000100:00100000:1.0:1775764719.685359:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2376:1862022098556672:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764719.689540:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098556928:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764719.689636:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098557184:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764719.689778:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098557056:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764719.701006:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098556928:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764719.701025:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098557184:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764719.701071:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098556800:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764719.701083:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098557056:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764719.701244:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2376:1862022098556672:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764721.459848:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119040384 00000100:00100000:1.0:1775764721.459861:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119040384 00000100:00100000:1.0:1775764721.459925:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 204 00000100:00100000:1.0:1775764721.459937:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dad500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:44493:x1862022119040384:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764721.459960:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741a00/0x73cfd86cfc17ba82 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e030513c8 expref: -99 pid: 61090 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764721.460067:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764721.460074:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dad500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:44493:x1862022119040384:12345-192.168.202.128@tcp:104: Request processed in 139us (354us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764721.460082:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 204 00000100:00100000:1.0:1775764721.502971:0:46140:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764721.503000:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119040512 00000100:00100000:1.0:1775764721.503013:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119040512 00000100:00100000:1.0:1775764721.503092:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 205 00000100:00100000:1.0:1775764721.503104:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23b9ff2450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42945:x1862022119040512:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764721.503125:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000009a0e5e19/0x73cfd86cfc17bb07 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x117:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e0305147e expref: -99 pid: 61422 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764721.512664:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764721.512691:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23b9ff2450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42945:x1862022119040512:12345-192.168.202.128@tcp:104: Request processed in 9578us (9969us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764721.512707:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 205 00000100:00100000:1.0:1775764721.523229:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200008:1.0:1775764721.626969:0:61428:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764721.626987:0:61428:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764721.627019:0:61428:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764721.627104:0:61428:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 61428 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764721.627114:0:61428:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61428 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764721.627125:0:61428:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61428 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764721.627156:0:61428:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61428:1862022098558976:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764721.627378:0:61428:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384904840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764721.690637:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119040768 00000100:00100000:1.0:1775764721.690650:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119040768 00000100:00100000:1.0:1775764721.690868:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 207 00000100:00100000:1.0:1775764721.690882:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dad880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43212:x1862022119040768:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764721.690902:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000137e66ca/0x73cfd86cfc17bb0e lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x117:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800420000020000 nid: local remote: 0x57b8ff0e03051485 expref: -99 pid: 61422 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00010000:00010000:1.0:1775764721.691022:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764721.691029:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dad880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43212:x1862022119040768:12345-192.168.202.128@tcp:104: Request processed in 152us (1318us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764721.691036:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 207 00000100:00100000:1.0:1775764721.720003:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119040896 00000100:00100000:1.0:1775764721.720015:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119040896 00000100:00100000:1.0:1775764721.720083:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 208 00000100:00100000:1.0:1775764721.720094:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dafb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42328:x1862022119040896:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764721.720118:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea800/0x73cfd86cfc17bb00 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03051470 expref: -99 pid: 61422 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1775764721.720272:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764721.720279:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dafb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42328:x1862022119040896:12345-192.168.202.128@tcp:104: Request processed in 188us (485us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764721.720287:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 208 00000100:00100000:1.0:1775764722.119441:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098562176:192.168.202.128@tcp:103:ldlm_bl.0 00000002:00010000:1.0:1775764722.250723:0:61461:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764722.250754:0:61461:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051493 expref: -99 pid: 61428 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764722.250772:0:61461:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051493 expref: -99 pid: 61428 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:1.0:1775764722.250780:0:61461:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764722.250789:0:61461:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764722.250795:0:61461:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bb15 00000080:00010000:1.0:1775764722.250803:0:61461:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb15 from it ffffa60b01e6bbb0 00010000:00010000:1.0:1775764722.250807:0:61461:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051493 expref: -99 pid: 61428 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764722.250814:0:61461:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051493 expref: -99 pid: 61428 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:1.0:1775764722.250836:0:61461:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764722.250850:0:61461:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:1.0:1775764722.250915:0:61461:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffce3733ab0 00000080:00200008:1.0:1775764722.250953:0:61461:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:2.0:1775764722.313488:0:61464:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f22.sanity-pcc, flags=265 00000080:00200000:2.0:1775764722.313493:0:61464:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), inode mode 81a4 mask 44 00000080:00200008:2.0:1775764722.313498:0:61464:0:(file.c:1057:ll_file_open()) START file f22.sanity-pcc:[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), flags 104000 00000002:00010000:2.0:1775764722.313511:0:61464:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240104001 00010000:00010000:2.0:1775764722.313560:0:61464:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bb3f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 61464 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764722.313567:0:61464:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bb3f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61464 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764722.313579:0:61464:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764722.313592:0:61464:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5500 x1862022098562432/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764722.313601:0:61464:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bb3f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61464 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764722.313635:0:61464:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61464:1862022098562432:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764722.313828:0:61464:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913863c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764722.320982:0:61464:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61464:1862022098562432:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764722.321026:0:61464:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bb3f lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051534 expref: -99 pid: 61464 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764722.321040:0:61464:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bb3f lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051534 expref: -99 pid: 61464 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764722.321055:0:61464:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bb3f lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051534 expref: -99 pid: 61464 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764722.321068:0:61464:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bb3f lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051534 expref: -99 pid: 61464 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764722.321088:0:61464:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23b9ea5500 x1862022098562432/t34359739053(34359739053) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764738 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764722.321104:0:61464:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea5500 x1862022098562432/t34359739053(34359739053) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764738 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764722.321120:0:61464:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bb3f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051534 expref: -99 pid: 61464 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764722.321136:0:61464:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bb3f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051534 expref: -99 pid: 61464 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764722.321147:0:61464:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/1 (0 0) 00000080:00200000:2.0:1775764722.321154:0:61464:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764722.321251:0:61464:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0x1:0x0], updating i_size 13 00000080:00200000:2.0:1775764722.321271:0:61464:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1:0x0]: layout version change: 1 -> 1 00000080:00010000:2.0:1775764722.321282:0:61464:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) for lock 0x73cfd86cfc17bb3f 00000080:00010000:2.0:1775764722.321289:0:61464:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb3f from it ffffa60b01eabbf0 00010000:00010000:2.0:1775764722.321294:0:61464:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bb3f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051534 expref: -99 pid: 61464 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764722.321305:0:61464:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bb3f lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051534 expref: -99 pid: 61464 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00100000:2.0:1775764722.321328:0:61464:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea5500 x1862022098562432/t34359739053(34359739053) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764738 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764722.321413:0:61464:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x1:0x0](ffff8b2390ec21c8) cmd=802066d3 arg=7ffd82f892f0 00800000:00000080:2.0:1775764722.321427:0:61464:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=802066d3 len=384 karg=ffff8b2392c00a00 uarg=0000000000000000 00000002:00000080:2.0:1775764722.321433:0:61464:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=802066d3 len=384 karg=ffff8b2392c00a00 uarg=0000000000000000 00000100:00100000:2.0:1775764722.321508:0:61464:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61464:1862022098562560:192.168.202.128@tcp:54:lfs.0 00000100:00100000:2.0:1775764722.321676:0:61464:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913863c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764722.351692:0:61464:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61464:1862022098562560:192.168.202.128@tcp:54:lfs.0 00000080:00200008:2.0:1775764722.351759:0:61464:0:(file.c:447:ll_file_release()) START file f22.sanity-pcc:[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), flags 104000 00010000:00010000:2.0:1775764722.351799:0:61464:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bb3f lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051534 expref: -99 pid: 61464 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764722.615615:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764723.158198:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200080:1.0:1775764723.163253:0:61475:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086696 arg=7fa5a0000b60 00000100:00100000:1.0:1775764723.163341:0:61475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098563200:192.168.202.128@tcp:900:lhsmtool_posix.0 00000100:00100000:1.0:1775764723.163510:0:61475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384904f00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764723.231155:0:61475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b8759300 going to sleep for 11 seconds 00000100:00100000:2.0:1775764723.240243:0:61475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098564096:192.168.202.128@tcp:53:lhsmtool_posix.0 00000002:00000080:2.0:1775764723.240286:0:61475:0:(mdc_request.c:1758:mdc_ioc_fid2path()) path got [0x200000007:0x1:0x0] from 18446744073709551615 #0: f22.sanity-pcc 00000080:00200000:2.0:1775764723.240499:0:61475:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764723.240520:0:61475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764723.240538:0:61475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051493 expref: -99 pid: 61428 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764723.240552:0:61475:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051493 expref: -99 pid: 61428 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:2.0:1775764723.240560:0:61475:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764723.240568:0:61475:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764723.240571:0:61475:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bb15 00000080:00010000:2.0:1775764723.240577:0:61475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb15 from it ffffa60b01eabbf8 00010000:00010000:2.0:1775764723.240581:0:61475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051493 expref: -99 pid: 61428 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764723.240590:0:61475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051493 expref: -99 pid: 61428 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:2.0:1775764723.240605:0:61475:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764723.240632:0:61475:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764723.240637:0:61475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764723.240642:0:61475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051493 expref: -99 pid: 61428 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1775764723.240650:0:61475:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051493 expref: -99 pid: 61428 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:2.0:1775764723.240656:0:61475:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764723.240659:0:61475:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764723.240662:0:61475:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bb15 00000080:00010000:2.0:1775764723.240665:0:61475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb15 from it ffffa60b01eabb38 00010000:00010000:2.0:1775764723.240668:0:61475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051493 expref: -99 pid: 61428 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1775764723.240675:0:61475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051493 expref: -99 pid: 61428 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1775764723.240682:0:61475:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:2.0:1775764723.240686:0:61475:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=. :VOLATILE:0000:19F5CE0E:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b23a00a7500, open_flags 280c2, mode 8180 opened 0 00000080:00200000:2.0:1775764723.240692:0:61475:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=. :VOLATILE:0000:19F5CE0E:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000002:00010000:2.0:1775764723.240712:0:61475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: . :VOLATILE:0000:19F5CE0E:fd=09,[0x200003ab4:0x3:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0500303 00010000:00010000:2.0:1775764723.240722:0:61475:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03051493 expref: -99 pid: 61428 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1775764723.240765:0:61475:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051493 expref: -99 pid: 61428 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1775764723.240771:0:61475:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764723.240774:0:61475:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17bb15 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051493 expref: -99 pid: 61428 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1775764723.240807:0:61475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 61475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.240814:0:61475:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61475 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764723.240825:0:61475:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764723.240834:0:61475:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4380 x1862022098564352/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1088/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764723.240843:0:61475:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61475 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764723.240876:0:61475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098564352:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764723.241153:0:61475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913863c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764723.246769:0:61475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098564352:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764723.246800:0:61475:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051588 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.246838:0:61475:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051588 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.246846:0:61475:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051588 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.246859:0:61475:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051588 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.246871:0:61475:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051588 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.246885:0:61475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051588 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.246894:0:61475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051588 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.246901:0:61475:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051588 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764723.246913:0:61475:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8b23b9ea4380 x1862022098564352/t34359739056(34359739056) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1088/664 e 0 to 0 dl 1775764739 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764723.246927:0:61475:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4380 x1862022098564352/t34359739056(34359739056) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1088/664 e 0 to 0 dl 1775764739 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764723.246942:0:61475:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051588 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764723.246955:0:61475:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051588 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.246966:0:61475:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/3 (0 0) 00000080:00200000:2.0:1775764723.246974:0:61475:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764723.246989:0:61475:0:(namei.c:1620:ll_create_it()) VFS Op:name=. :VOLATILE:0000:19F5CE0E:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000080:00200000:2.0:1775764723.247025:0:61475:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab4:0x3:0x0](ffff8b2390ec1988) 00000080:00200000:2.0:1775764723.247173:0:61475:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x3:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1775764723.247185:0:61475:0:(namei.c:1586:ll_create_node()) setting l_ast_data to inode [0x200003ab4:0x3:0x0](ffff8b2390ec1988) 00000080:00010000:2.0:1775764723.247189:0:61475:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x3:0x0](ffff8b2390ec1988) for lock 0x73cfd86cfc17bb54 00000080:00010000:2.0:1775764723.247198:0:61475:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x3:0x0](ffff8b2390ec1988) 00000080:00200008:2.0:1775764723.247211:0:61475:0:(file.c:1057:ll_file_open()) START file . :VOLATILE:0000:19F5CE0E:fd=09:[0x200003ab4:0x3:0x0](ffff8b2390ec1988), flags 500302 00000002:00100000:2.0:1775764723.247229:0:61475:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4380 x1862022098564352/t34359739056(34359739056) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 608/664 e 0 to 0 dl 1775764739 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764723.247271:0:61475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb54 from it ffff8b238504c480 00010000:00010000:2.0:1775764723.247276:0:61475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051588 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.247284:0:61475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051588 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764723.247327:0:61475:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x3:0x0](ffff8b2390ec1988), inode mode 8180 mask 40 00000080:00200000:2.0:1775764723.247379:0:61475:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764723.247394:0:61475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764723.247436:0:61475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 61475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.247444:0:61475:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.247488:0:61475:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61475 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764723.247510:0:61475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098564480:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764723.247594:0:61475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386e40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764723.253933:0:61475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098564480:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764723.253977:0:61475:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305159d expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.253990:0:61475:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305159d expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.254003:0:61475:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305159d expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.254013:0:61475:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305159d expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.254028:0:61475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305159d expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.254038:0:61475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305159d expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.254051:0:61475:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305159d expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764723.254064:0:61475:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea5180 x1862022098564480/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764739 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764723.254080:0:61475:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305159d expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.254095:0:61475:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764723.254101:0:61475:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764723.254134:0:61475:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764723.254140:0:61475:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bb5b 00000080:00010000:2.0:1775764723.254146:0:61475:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764723.254149:0:61475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb5b from it ffffa60b01eabc10 00010000:00010000:2.0:1775764723.254152:0:61475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305159d expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.254160:0:61475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305159d expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764723.254183:0:61475:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764723.254201:0:61475:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=. :VOLATILE:0000:19F5CE0E:fd=09, flags=0 00000080:00200000:2.0:1775764723.254207:0:61475:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764723.254221:0:61475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764723.254229:0:61475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305159d expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.254244:0:61475:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305159d expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764723.254251:0:61475:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764723.254255:0:61475:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764723.254257:0:61475:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bb5b 00000080:00010000:2.0:1775764723.254260:0:61475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb5b from it ffffa60b01eabd70 00010000:00010000:2.0:1775764723.254263:0:61475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305159d expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.254270:0:61475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305159d expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764723.254277:0:61475:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:2.0:1775764723.254282:0:61475:0:(namei.c:2405:ll_unlink()) VFS Op:name=. :VOLATILE:0000:19F5CE0E:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08) 00010000:00010000:2.0:1775764723.254354:0:61475:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0305159d expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764723.254380:0:61475:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03051588 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764723.254412:0:61475:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x3:0x0]: losing layout lock 00010000:00010000:2.0:1775764723.254489:0:61475:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051588 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.254497:0:61475:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305159d expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764723.254507:0:61475:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1775764723.254509:0:61475:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb54 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051588 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.254528:0:61475:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb5b lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305159d expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000020:00100000:2.0:1775764723.254546:0:61475:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764723.254553:0:61475:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5180 x1862022098564736/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/1032 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764723.254579:0:61475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098564736:192.168.202.128@tcp:36:lhsmtool_posix.0 00000100:00100000:2.0:1775764723.254671:0:61475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386e40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764723.258546:0:61475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098564736:192.168.202.128@tcp:36:lhsmtool_posix.0 00000002:00010000:2.0:1775764723.258660:0:61475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x0:0x0:0x0]) in obj [0x200003ab4:0x3:0x0], intent: getxattr flags 0 00010000:00010000:2.0:1775764723.258725:0:61475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bb62 lrc: 3/1,0 mode: --/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 61475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.258734:0:61475:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bb62 lrc: 3/1,0 mode: --/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.258744:0:61475:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bb62 lrc: 3/1,0 mode: --/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61475 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764723.258763:0:61475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098564864:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764723.258862:0:61475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386e40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764723.266407:0:61475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098564864:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764723.266436:0:61475:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bb62 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515b9 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.266499:0:61475:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bb62 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030515b9 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.266511:0:61475:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bb62 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515b9 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764723.266526:0:61475:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=80 disposition=0, status=0 req@ffff8b23b9ea5180 x1862022098564864/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 568/1072 e 0 to 0 dl 1775764739 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764723.266537:0:61475:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bb62 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515b9 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.266550:0:61475:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/3 (0 0) 00000080:00200000:2.0:1775764723.266557:0:61475:0:(namei.c:953:ll_intent_lock()) intent lock 128 on i1 [0x200003ab4:0x3:0x0] suppgids 0 -1: rc 0 00000080:00010000:2.0:1775764723.266580:0:61475:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x3:0x0](ffff8b2390ec1988) for lock 0x73cfd86cfc17bb62 00000080:00010000:2.0:1775764723.266588:0:61475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb62 from it ffffa60b01eabc28 00010000:00010000:2.0:1775764723.266592:0:61475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bb62 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030515b9 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.266601:0:61475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bb62 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515b9 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1775764723.266647:0:61475:0:(llite_lib.c:2274:ll_setattr_raw()) START file . :VOLATILE:0000:19F5CE0E:fd=09:[0x200003ab4:0x3:0x0](ffff8b2390ec1988) current size 0, valid attrs 1846, mode 8b23, uid 0, gid 0, new size 0, atime 1775764723.000000000, mtime 1775764723.000000000, ctime 1775764723.000000000 00000020:00100000:2.0:1775764723.266695:0:61475:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764723.266701:0:61475:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5180 x1862022098565120/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764723.266722:0:61475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098565120:192.168.202.128@tcp:36:lhsmtool_posix.0 00000100:00100000:2.0:1775764723.266800:0:61475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386e40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764723.281989:0:61475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098565120:192.168.202.128@tcp:36:lhsmtool_posix.0 00010000:00010000:2.0:1775764723.282066:0:61475:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 31 res 8589949620/3 (0 0) 00000080:00010000:2.0:1775764723.282084:0:61475:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x3:0x0](ffff8b2390ec1988) 00000002:00010000:2.0:1775764723.282099:0:61475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x3:0x0]) in obj [0x200003ab4:0x3:0x0], intent: layout flags 0 00010000:00010000:2.0:1775764723.282172:0:61475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 61475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.282182:0:61475:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.282195:0:61475:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61475 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764723.282219:0:61475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098565248:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764723.282350:0:61475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386e40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764723.291530:0:61475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098565248:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764723.291562:0:61475:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.291577:0:61475:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764723.291596:0:61475:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea6680 x1862022098565248/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/400 e 0 to 0 dl 1775764739 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764723.291606:0:61475:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764723.291633:0:61475:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.291644:0:61475:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/3 (0 0) 00000080:00200000:2.0:1775764723.291650:0:61475:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x3:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764723.291667:0:61475:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x3:0x0](ffff8b2390ec1988) for lock 0x73cfd86cfc17bb69 00000080:00010000:2.0:1775764723.291675:0:61475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb69 from it ffffa60b01eaba68 00010000:00010000:2.0:1775764723.291679:0:61475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.291688:0:61475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.291712:0:61475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.291721:0:61475:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764723.291729:0:61475:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x3:0x0](0000000042b87f29) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764723.291809:0:61475:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x3:0x0]: layout version change: 4294967294 -> 0 00010000:00010000:2.0:1775764723.291831:0:61475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.291838:0:61475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764723.291868:0:61475:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x3:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764723.291882:0:61475:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764723.306641:0:61475:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x3:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1775764723.306674:0:61475:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file . :VOLATILE:0000:19F5CE0E:fd=09:[0x200003ab4:0x3:0x0](ffff8b2390ec1988) current size 0, valid attrs 1846, mode 8b23, uid 0, gid 0, new size 0, atime 1775764723.000000000, mtime 1775764723.000000000, ctime 1775764723.000000000, rc 0 00000080:00200008:2.0:1775764723.306747:0:61475:0:(file.c:447:ll_file_release()) START file 0x200003ab4:0x1:0x0:[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), flags 1504000 00000080:00200080:2.0:1775764723.306798:0:61475:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=400866d6 arg=7fa5a0000b78 00800000:00000080:2.0:1775764723.306807:0:61475:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b01eabcf0 uarg=0000000000000000 00000002:00000080:2.0:1775764723.306814:0:61475:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b01eabcf0 uarg=0000000000000000 00000020:00100000:2.0:1775764723.306859:0:61475:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 57, max 7 00000100:00100000:2.0:1775764723.306876:0:61475:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5880 x1862022098565632/t0(0) o57->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764723.306906:0:61475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098565632:192.168.202.128@tcp:57:lhsmtool_posix.0 00000100:00100000:2.0:1775764723.307021:0:61475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386e40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764723.322882:0:61475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098565632:192.168.202.128@tcp:57:lhsmtool_posix.0 00000080:00200008:2.0:1775764723.323080:0:61475:0:(file.c:6226:ll_getattr_dentry()) START file . :VOLATILE:0000:19F5CE0E:fd=09:[0x200003ab4:0x3:0x0](ffff8b2390ec1988), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764723.323089:0:61475:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x3:0x0](ffff8b2390ec1988),name=. :VOLATILE:0000:19F5CE0E:fd=09 00000002:00010000:2.0:1775764723.323104:0:61475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x3:0x0]) in obj [0x200003ab4:0x3:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764723.323118:0:61475:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 30 res 8589949620/3 (0 0) 00010000:00010000:2.0:1775764723.323161:0:61475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb70 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 61475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.323170:0:61475:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb70 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 61475 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.323179:0:61475:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb70 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 61475 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764723.323199:0:61475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098565760:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764723.323273:0:61475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386e40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764723.326019:0:61475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098565760:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764723.326048:0:61475:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb70 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515d5 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.326061:0:61475:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb70 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515d5 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.326072:0:61475:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb70 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030515d5 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.326083:0:61475:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb70 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515d5 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.326096:0:61475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb70 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030515d5 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.326106:0:61475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb70 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030515d5 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.326113:0:61475:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb70 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030515d5 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764723.326127:0:61475:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea5880 x1862022098565760/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/640 e 0 to 0 dl 1775764739 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764723.326140:0:61475:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb70 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515d5 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764723.326156:0:61475:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb70 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515d5 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.326170:0:61475:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/3 (0 0) 00000080:00200000:2.0:1775764723.326177:0:61475:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200003ab4:0x3:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764723.326287:0:61475:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x3:0x0]: layout version change: 0 -> 0 00000080:00010000:2.0:1775764723.326299:0:61475:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x3:0x0](ffff8b2390ec1988) for lock 0x73cfd86cfc17bb70 00000080:00010000:2.0:1775764723.326305:0:61475:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x3:0x0](ffff8b2390ec1988) 00000080:00010000:2.0:1775764723.326309:0:61475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb70 from it ffffa60b01eabcd8 00010000:00010000:2.0:1775764723.326313:0:61475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb70 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030515d5 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.326321:0:61475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb70 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030515d5 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764723.326402:0:61475:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x3:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1775764723.326412:0:61475:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x3:0x0] 00000008:00010000:2.0:1775764723.326443:0:61475:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764723.326447:0:61475:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e240b0, osc lock ffff8b238a3f8d10, flags 0x400001002 00010000:00010000:2.0:1775764723.326546:0:61475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bb77 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.326557:0:61475:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bb77 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1775764723.326601:0:61475:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1775764723.336752:0:61475:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x3:0x0] updating i_size 0 i_blocks 1 00000020:00010000:2.0:1775764723.336764:0:61475:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027e238 (ffff8b23812ecd88/0) at cl_lock_release():245 00000020:00010000:2.0:1775764723.336770:0:61475:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027e238 (ffff8b23812ecd88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764723.336776:0:61475:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e240b0 (ffff8b23812ecab8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764723.336782:0:61475:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027e238 (ffff8b23812ecd88/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764723.336786:0:61475:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e240b0 (ffff8b23812ecd88/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764723.336844:0:61475:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x3:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:2.0:1775764723.336943:0:61475:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=401866d8 arg=7fa5a7a76910 00800000:00000080:2.0:1775764723.336955:0:61475:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b01eabcf0 uarg=0000000000000000 00000002:00000080:2.0:1775764723.336964:0:61475:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b01eabcf0 uarg=0000000000000000 00000020:00100000:2.0:1775764723.336999:0:61475:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 57, max 7 00000100:00100000:2.0:1775764723.337010:0:61475:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4a80 x1862022098566144/t0(0) o57->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764723.337039:0:61475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098566144:192.168.202.128@tcp:57:lhsmtool_posix.0 00000100:00100000:2.0:1775764723.337166:0:61475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386e40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764723.341988:0:61475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098566144:192.168.202.128@tcp:57:lhsmtool_posix.0 00000080:00200008:2.0:1775764723.342277:0:61475:0:(file.c:2612:do_file_write_iter()) START file . :VOLATILE:0000:19F5CE0E:fd=09:[0x200003ab4:0x3:0x0], ppos: 0, count: 13 00000080:00200000:2.0:1775764723.342309:0:61475:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 13 bytes 00000080:00200000:2.0:1775764723.342329:0:61475:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 13 00000080:00200000:2.0:1775764723.342374:0:61475:0:(file.c:2058:ll_file_io_generic()) . :VOLATILE:0000:19F5CE0E:fd=09: write ppos: 0, bytes: 13 00000080:00200000:2.0:1775764723.342380:0:61475:0:(file.c:1815:ll_io_set_mirror()) . :VOLATILE:0000:19F5CE0E:fd=09: desiginated mirror: 0 00000020:00200000:2.0:1775764723.343589:0:61475:0:(cl_io.c:214:cl_io_rw_init()) header@000000000106900a[0x110, 1, [0x200003ab4:0x3:0x0] hash] 00000020:00200000:2.0:1775764723.343597:0:61475:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 13) 0 0 00000080:00200000:2.0:1775764723.343604:0:61475:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x3:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764723.343785:0:61475:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:2.0:1775764723.343796:0:61475:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 13) 13, 13 00020000:00200000:2.0:1775764723.343801:0:61475:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764723.343858:0:61475:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1775764723.343875:0:61475:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764723.343882:0:61475:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e240b0, osc lock ffff8b238a3f8d10, flags 0x0 00010000:00010000:2.0:1775764723.343930:0:61475:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b23ab6dd800 type 11 mode 2 res 9663677440/280 (0 4095) 00010000:00010000:2.0:1775764723.344173:0:61475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 3/0,1 mode: --/PW res: [0x240000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.344185:0:61475:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 3/0,1 mode: --/PW res: [0x240000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.344194:0:61475:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 3/0,1 mode: --/PW res: [0x240000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764723.344242:0:61475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098566272:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764723.344440:0:61475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386e40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764723.363892:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119042176 00000100:00100000:1.0:1775764723.363914:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119042176 00000100:00100000:1.0:1775764723.364333:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 210 00000100:00100000:1.0:1775764723.364346:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dace00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42944:x1862022119042176:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764723.364365:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bb77 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e030515dc expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764723.364495:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764723.364502:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dace00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42944:x1862022119042176:12345-192.168.202.128@tcp:104: Request processed in 159us (3899us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764723.364510:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 210 00010000:00010000:2.0:1775764723.364573:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764723.364581:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bb77 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e030515dc expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.364590:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6eb200 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:2.0:1775764723.364596:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bb77 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x57b8ff0e030515dc expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764723.364669:0:46149:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x3:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764723.364683:0:46149:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x3:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775764723.364761:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bb77 lrc: 2/0,0 mode: --/PR res: [0x240000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e030515dc expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.364770:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764723.364794:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bb77 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e030515dc expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.364802:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bb77 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e030515dc expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.364818:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764723.364820:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764723.364822:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764723.372025:0:46139:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764723.372047:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119042304 00000100:00100000:1.0:1775764723.372058:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119042304 00000100:00100000:1.0:1775764723.372237:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 211 00000100:00100000:1.0:1775764723.372249:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2392f61850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42310:x1862022119042304:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764723.372265:0:46139:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 4/0,1 mode: --/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764723.372281:0:46139:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 4/0,1 mode: --/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764723.372290:0:46139:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 4/0,1 mode: --/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764723.372303:0:46139:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764723.372314:0:46139:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764723.372327:0:46139:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000000000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764723.372346:0:46139:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b23ac6eaa00) 00000100:00100000:1.0:1775764723.372416:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2392f61850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42310:x1862022119042304:12345-192.168.202.128@tcp:105: Request processed in 169us (510us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764723.372423:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 211 00000100:00100000:2.0:1775764723.377799:0:61475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098566272:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764723.377835:0:61475:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.377862:0:61475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1775764723.377876:0:61475:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000020000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.377888:0:61475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.377895:0:61475:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764723.377927:0:61475:0:(vvp_io.c:1286:vvp_io_write_start()) . :VOLATILE:0000:19F5CE0E:fd=09: write [0, 13) append 0 append lockpos 0 00000080:00200000:2.0:1775764723.377945:0:61475:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 13 bytes 00000080:00200000:2.0:1775764723.378009:0:61475:0:(rw26.c:858:ll_write_end()) pos 0, len 13, copied 13 00000080:00200000:2.0:1775764723.378019:0:61475:0:(rw26.c:916:ll_write_end()) page@00000000d1ce3a9a[3 00000000e90eb546 2 1 00000000dd29be51] 00000080:00200000:2.0:1775764723.378023:0:61475:0:(rw26.c:916:ll_write_end()) vmpage @0000000006671fbe 17ffffc0001001 3:0 ffff8b23baeaec40 0 lru 00000080:00200000:2.0:1775764723.378030:0:61475:0:(rw26.c:916:ll_write_end()) osc-page@00000000c66b2839 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 000000003596ffcd 00000000eee656ac > 3< 0 0 > 4< 0 0 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764723.378032:0:61475:0:(rw26.c:916:ll_write_end()) end page@00000000d1ce3a9a 00000080:00200000:2.0:1775764723.378034:0:61475:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764723.378039:0:61475:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 13 prio 0 00000008:00010000:2.0:1775764723.378060:0:61475:0:(osc_cache.c:848:osc_extent_find()) ### extent: 0000000084a3a955 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764723.378080:0:61475:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2390ec1b38, count 1, dirtied 1 00000080:00200000:2.0:1775764723.378084:0:61475:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 13 bytes, tot: 13 00000080:00200000:2.0:1775764723.378088:0:61475:0:(vvp_io.c:1353:vvp_io_write_start()) . :VOLATILE:0000:19F5CE0E:fd=09: write bytes 0, result: 13 00000020:00010000:2.0:1775764723.379882:0:61475:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36bc68 (ffff8b23812ecd88/0) at cl_lock_release():245 00000020:00010000:2.0:1775764723.379887:0:61475:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36bc68 (ffff8b23812ecd88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764723.379892:0:61475:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e240b0 (ffff8b23812ecab8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764723.379902:0:61475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.379915:0:61475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1775764723.379933:0:61475:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36bc68 (ffff8b23812ecd88/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764723.379936:0:61475:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e240b0 (ffff8b23812ecd88/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764723.379944:0:61475:0:(vvp_io.c:517:vvp_io_advance()) advancing 13 bytes 00000080:00200000:2.0:1775764723.379951:0:61475:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:2.0:1775764723.379985:0:61475:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x3:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764723.379989:0:61475:0:(file.c:2206:ll_file_io_generic()) . :VOLATILE:0000:19F5CE0E:fd=09: 2 io complete with rc: 0, result: 13, restart: 0 00000080:00200000:2.0:1775764723.380002:0:61475:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x3:0x0] updating i_size 13 i_blocks 1 00000080:00200000:2.0:1775764723.380007:0:61475:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 13 00000080:00200000:2.0:1775764723.380239:0:61475:0:(file.c:5304:ll_fsync()) VFS Op:inode=[0x200003ab4:0x3:0x0](ffff8b2390ec1988), start 0, end 9223372036854775807, datasync 0 00000080:00200000:2.0:1775764723.380270:0:61475:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x3:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764723.380279:0:61475:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:2.0:1775764723.380305:0:61475:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 0000000084a3a955 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1775764723.380318:0:61475:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 0000000084a3a955 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.380365:0:61475:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 1 00000008:00100000:2.0:1775764723.380438:0:61475:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea4a80 - object 0x240000400:280 offset 0<>13 00000080:00200000:2.0:1775764723.400071:0:61475:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x3:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:2.0:1775764723.400143:0:61475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098566656:192.168.202.128@tcp:44:lhsmtool_posix.0 00000100:00100000:2.0:1775764723.400230:0:61475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386e40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764723.519661:0:61475:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 34359739057 gen 1 00000100:00100000:2.0:1775764723.519678:0:61475:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23864f0700 x1862022098557568/t34359739034(34359739034) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/4688 e 0 to 0 dl 1775764737 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764723.519708:0:61475:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23b9ea5500 x1862022098562432/t34359739053(34359739053) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764738 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764723.519718:0:61475:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23b9ea4380 x1862022098564352/t34359739056(34359739056) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 608/664 e 0 to 0 dl 1775764739 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764723.519734:0:61475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098566656:192.168.202.128@tcp:44:lhsmtool_posix.0 00000080:00200000:2.0:1775764723.519783:0:61475:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x3:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764723.519794:0:61475:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764723.635244:0:61475:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x3:0x0] updating i_size 13 i_blocks 11 00000080:00200000:2.0:1775764723.635302:0:61475:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x3:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:2.0:1775764723.635331:0:61475:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x3:0x0](ffff8b2390ec1988) cmd=c03066b0 arg=7fa5a7a75ac0 00000080:00200008:2.0:1775764723.635342:0:61475:0:(llite_lib.c:2274:ll_setattr_raw()) START file . :VOLATILE:0000:19F5CE0E:fd=09:[0x200003ab4:0x3:0x0](ffff8b2390ec1988) current size 13, valid attrs 1f0, mode 0, uid 0, gid 0, new size 0, atime 1775764721.000000000, mtime 1775764721.000000000, ctime 1775764721.000000000 00010000:00010000:2.0:1775764723.635375:0:61475:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb70 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e030515d5 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.635404:0:61475:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764723.635447:0:61475:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb70 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030515d5 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.635487:0:61475:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764723.635490:0:61475:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bb70 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030515d5 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000020:00100000:2.0:1775764723.635509:0:61475:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764723.635520:0:61475:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5180 x1862022098567040/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764723.635541:0:61475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098567040:192.168.202.128@tcp:36:lhsmtool_posix.0 00000100:00100000:2.0:1775764723.635656:0:61475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386e40 going to sleep for 11 seconds 00010000:00010000:2.0:1775764723.639889:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764723.660672:0:61475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098567040:192.168.202.128@tcp:36:lhsmtool_posix.0 00000080:00200000:2.0:1775764723.660767:0:61475:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x3:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764723.660779:0:61475:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764723.673195:0:61475:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x3:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00800000:00000080:2.0:1775764723.673225:0:61475:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d4 len=384 karg=ffff8b238a3c2a00 uarg=0000000000000000 00000002:00000080:2.0:1775764723.673237:0:61475:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d4 len=384 karg=ffff8b238a3c2a00 uarg=0000000000000000 00000020:00100000:2.0:1775764723.673267:0:61475:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 55, max 7 00000100:00100000:2.0:1775764723.673279:0:61475:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6300 x1862022098567424/t0(0) o55->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 472/224 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764723.673318:0:61475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098567424:192.168.202.128@tcp:55:lhsmtool_posix.0 00000100:00100000:2.0:1775764723.673413:0:61475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386e40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764723.681567:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764723.682098:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764723.682106:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bb62 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e030515b9 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.682116:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6ea000 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764723.682121:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bb62 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e030515b9 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.682130:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bb62 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e030515b9 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.682139:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bb62 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x57b8ff0e030515b9 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.682199:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bb62 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030515b9 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.682206:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764723.682237:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bb62 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030515b9 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.682245:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bb62 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x3:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030515b9 expref: -99 pid: 61475 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.682262:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764723.682264:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764723.682266:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764723.696404:0:61475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098567424:192.168.202.128@tcp:55:lhsmtool_posix.0 00000080:00200008:2.0:1775764723.696495:0:61475:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file . :VOLATILE:0000:19F5CE0E:fd=09:[0x200003ab4:0x3:0x0](ffff8b2390ec1988) current size 13, valid attrs 1f0, mode 0, uid 0, gid 0, new size 0, atime 1775764721.000000000, mtime 1775764721.000000000, ctime 1775764721.000000000, rc 0 00000080:00200080:2.0:1775764723.696516:0:61475:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=400866d7 arg=7fa5a0000b78 00000080:00200000:2.0:1775764723.696565:0:61475:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x3:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764723.696578:0:61475:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:1.0:1775764723.701778:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119042816 00000100:00100000:1.0:1775764723.701790:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119042816 00000100:00100000:1.0:1775764723.701851:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 213 00000100:00100000:1.0:1775764723.701872:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dad500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42944:x1862022119042816:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764723.701890:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800420000020000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 1 00010000:00010000:1.0:1775764723.701976:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764723.701983:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dad500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42944:x1862022119042816:12345-192.168.202.128@tcp:104: Request processed in 123us (336us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764723.701991:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 213 00010000:00010000:2.0:1775764723.702074:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764723.702082:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800420000020000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 1 00010000:00010000:2.0:1775764723.702092:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6eaa00 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:2.0:1775764723.702100:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800428400020000 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 1 00000080:00200000:2.0:1775764723.702167:0:46150:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x3:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764723.702235:0:46150:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x3:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775764723.702307:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 2/0,0 mode: --/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 1 00010000:00010000:2.0:1775764723.702315:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764723.702345:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 1/0,0 mode: --/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 1 00010000:00010000:2.0:1775764723.702354:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bb7e lrc: 0/0,0 mode: --/PW res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e030515e3 expref: -99 pid: 61475 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 1 00010000:00010000:2.0:1775764723.702382:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764723.702384:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764723.702386:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00800000:00000080:2.0:1775764723.719864:0:61475:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b01eabcf0 uarg=0000000000000000 00000002:00000080:2.0:1775764723.719876:0:61475:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b01eabcf0 uarg=0000000000000000 00000020:00100000:2.0:1775764723.719905:0:61475:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 57, max 7 00000100:00100000:2.0:1775764723.719914:0:61475:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4000 x1862022098568064/t0(0) o57->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764723.719939:0:61475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098568064:192.168.202.128@tcp:57:lhsmtool_posix.0 00000100:00100000:2.0:1775764723.720029:0:61475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386e40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764723.723428:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119042944 00000100:00100000:1.0:1775764723.723439:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119042944 00000100:00100000:1.0:1775764723.723525:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 214 00000100:00100000:1.0:1775764723.723535:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dadc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42326:x1862022119042944:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764723.723552:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764723.723735:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764723.723743:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dadc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42326:x1862022119042944:12345-192.168.202.128@tcp:104: Request processed in 209us (461us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764723.723752:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 214 00010000:00010000:2.0:1775764723.731090:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764723.731104:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764723.731117:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6ea600 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764723.731124:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764723.731132:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764723.731141:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 3/0,0 mode: CR/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764723.731209:0:46149:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x3:0x0]: losing layout lock 00010000:00010000:2.0:1775764723.731282:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 2/0,0 mode: --/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764723.731290:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764723.731330:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764723.731338:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb69 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x3:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e030515c7 expref: -99 pid: 61475 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764723.731363:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764723.731365:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764723.731367:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764723.747508:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764723.748329:0:61474:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.748341:0:61474:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764723.748360:0:61474:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b2382daf480 x1862022098563072/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/344 e 0 to 0 dl 1775764738 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764723.748372:0:61474:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.748384:0:61474:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/1 (0 0) 00000080:00200000:2.0:1775764723.748391:0:61474:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x1:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764723.748409:0:61474:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) for lock 0x73cfd86cfc17bb46 00000080:00010000:2.0:1775764723.748418:0:61474:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb46 from it ffffa60b01e6bbe8 00010000:00010000:2.0:1775764723.748422:0:61474:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.748432:0:61474:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.748494:0:61474:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.748505:0:61474:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764723.748512:0:61474:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1:0x0](00000000492670d3) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:2.0:1775764723.748563:0:61474:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61474:1862022098568320:192.168.202.128@tcp:49:lfs.0 00000100:00100000:2.0:1775764723.748630:0:61474:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386f00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764723.753045:0:61474:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61474:1862022098568320:192.168.202.128@tcp:49:lfs.0 00010000:00010000:2.0:1775764723.753186:0:61474:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.753199:0:61474:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764723.753232:0:61474:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764723.753240:0:61474:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764723.753276:0:61474:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764723.753293:0:61474:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764723.753301:0:61474:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1:0x0](00000000492670d3) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764723.753341:0:61474:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764723.753346:0:61474:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764723.753410:0:61474:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1:0x0]: layout version change: 4294967294 -> 2 00010000:00010000:2.0:1775764723.753419:0:61474:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764723.753427:0:61474:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00010000:2.0:1775764723.753691:0:61474:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) 00000002:00010000:2.0:1775764723.753704:0:61474:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1:0x0]) in obj [0x200003ab4:0x1:0x0], intent: layout flags 02 00010000:00010000:2.0:1775764723.753724:0:61474:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764723.753755:0:61474:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1:0x0]: losing layout lock 00010000:00010000:2.0:1775764723.753814:0:61474:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764723.753821:0:61474:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764723.753823:0:61474:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bb46 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051550 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764723.753864:0:61474:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 61474 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.753871:0:61474:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61474 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764723.753883:0:61474:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 2 is allocated opc 101, max 7 00000100:00100000:2.0:1775764723.753894:0:61474:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4e00 x1862022098568576/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/880 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764723.753904:0:61474:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61474 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764723.753925:0:61474:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61474:1862022098568576:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764723.754006:0:61474:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764723.761043:0:61474:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61474:1862022098568576:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764723.761069:0:61474:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 4/0,1 mode: --/EX res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.761081:0:61474:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 4/0,1 mode: --/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.761093:0:61474:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.761102:0:61474:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.761118:0:61474:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.761125:0:61474:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.761131:0:61474:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764723.761153:0:61474:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea4e00 x1862022098568576/t34359739065(34359739065) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/600 e 0 to 0 dl 1775764739 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764723.761170:0:61474:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764723.761181:0:61474:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.761192:0:61474:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/1 (0 0) 00000080:00200000:2.0:1775764723.761198:0:61474:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x1:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764723.761206:0:61474:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) for lock 0x73cfd86cfc17bb85 00000080:00010000:2.0:1775764723.761213:0:61474:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb85 from it ffffa60b01e6bc10 00010000:00010000:2.0:1775764723.761216:0:61474:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.761224:0:61474:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.761248:0:61474:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.761258:0:61474:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764723.761267:0:61474:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1:0x0](00000000492670d3) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.761355:0:61474:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764723.761363:0:61474:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764723.761392:0:61474:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764723.761404:0:61474:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764723.761447:0:61474:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764723.761661:0:61474:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764723.761672:0:61474:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764723.761679:0:61474:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1:0x0](00000000492670d3) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764723.761723:0:61474:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764723.761728:0:61474:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764723.761759:0:61474:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764723.762065:0:61474:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1:0x0]: layout version change: 4294967294 -> 3 00010000:00010000:2.0:1775764723.762076:0:61474:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764723.762085:0:61474:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:2.0:1775764723.765728:0:61474:0:(file.c:2454:do_file_read_iter()) START file f22.sanity-pcc:[0x200003ab4:0x1:0x0], ppos: 0, count: 1048576 00000100:00100000:1.0:1775764723.782827:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764723.782955:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119043456 00000100:00100000:1.0:1775764723.782963:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119043456 00000100:00100000:1.0:1775764723.783009:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 216 00000100:00100000:1.0:1775764723.783017:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23864fd500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42326:x1862022119043456:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764723.783044:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764723.783100:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764723.783106:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23864fd500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42326:x1862022119043456:12345-192.168.202.128@tcp:104: Request processed in 90us (282us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764723.783112:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 216 00010000:00010000:1.0:1775764723.783145:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764723.783151:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764723.783160:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6ea600 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764723.783165:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0xa rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764723.783173:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0xa rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764723.783181:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 3/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0xa rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764723.783222:0:46150:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1:0x0]: losing layout lock 00010000:00010000:1.0:1775764723.783270:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 2/0,0 mode: --/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0xa rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764723.783277:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764723.783299:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0xa rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764723.783306:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea600/0x73cfd86cfc17bb85 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xa/0xa rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03051622 expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764723.783327:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764723.783329:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764723.783331:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764723.783343:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098568832:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764723.793478:0:61475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098568064:192.168.202.128@tcp:57:lhsmtool_posix.0 00000080:00200008:1.0:1775764723.793741:0:61475:0:(file.c:447:ll_file_release()) START file . :VOLATILE:0000:19F5CE0E:fd=09:[0x200003ab4:0x3:0x0](ffff8b2390ec1988), flags 500002 00000002:00100000:1.0:1775764723.793815:0:61475:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea4380 x1862022098564352/t34359739056(34359739056) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 608/664 e 0 to 0 dl 1775764739 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764723.793827:0:61475:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764723.793832:0:61475:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ff100 x1862022098569088/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764723.793853:0:61475:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098569088:192.168.202.128@tcp:35:lhsmtool_posix.0 00000100:00100000:1.0:1775764723.793929:0:61475:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384904840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764723.805773:0:61474:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b8759180 going to sleep for 11 seconds 00000100:00100000:1.0:1775764723.808974:0:61475:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61475:1862022098569088:192.168.202.128@tcp:35:lhsmtool_posix.0 00000002:00100000:1.0:1775764723.809008:0:61475:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea4380 x1862022098564352/t34359739056(34359739056) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 608/664 e 0 to 0 dl 1775764739 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764723.812220:0:61474:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61474:1862022098569216:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764723.812252:0:61474:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764723.812268:0:61474:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764723.812283:0:61474:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23864f1180 x1862022098569216/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/600 e 0 to 0 dl 1775764739 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764723.812375:0:61474:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764723.812390:0:61474:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764723.812406:0:61474:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/1 (0 0) 00000080:00200000:1.0:1775764723.812414:0:61474:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x1:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1775764723.812435:0:61474:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) for lock 0x73cfd86cfc17bb93 00000080:00010000:1.0:1775764723.812444:0:61474:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb93 from it ffffa60b01e6b7e8 00010000:00010000:1.0:1775764723.812449:0:61474:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764723.812489:0:61474:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764723.812594:0:61474:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764723.812604:0:61474:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:1.0:1775764723.812622:0:61474:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1:0x0](00000000492670d3) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764723.812705:0:61474:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1:0x0]: layout version change: 4294967294 -> 3 00010000:00010000:1.0:1775764723.812729:0:61474:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764723.812737:0:61474:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00020000:00200000:1.0:1775764723.812760:0:61474:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x1:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 1, mirror count: 2 00020000:00200000:1.0:1775764723.812763:0:61474:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:1.0:1775764723.812863:0:61474:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:1.0:1775764723.812868:0:61474:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764723.812900:0:61474:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:1.0:1775764723.812917:0:61474:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764723.812922:0:61474:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5fb0, osc lock ffff8b2382cf7300, flags 0x400000000000000 00010000:00010000:1.0:1775764723.812947:0:61474:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000e34c7ac2/0x73cfd86cfc17bb8c lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051630 expref: -99 pid: 61474 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764723.812959:0:61474:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000e34c7ac2/0x73cfd86cfc17bb8c lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051630 expref: -99 pid: 61474 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764723.812971:0:61474:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000e34c7ac2/0x73cfd86cfc17bb8c lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051630 expref: -99 pid: 61474 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764723.812978:0:61474:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000e34c7ac2/0x73cfd86cfc17bb8c lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051630 expref: -99 pid: 61474 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764723.812985:0:61474:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000e34c7ac2/0x73cfd86cfc17bb8c lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051630 expref: -99 pid: 61474 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764723.812993:0:61474:0:(vvp_io.c:826:vvp_io_read_start()) f22.sanity-pcc: read [0, 1048576) 00000001:00010000:1.0:1775764723.813008:0:61474:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x1:0x0] 00000008:00010000:1.0:1775764723.813013:0:61474:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764723.813015:0:61474:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5130, osc lock ffff8b2382cf6850, flags 0x400000400001002 00010000:00010000:1.0:1775764723.813022:0:61474:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000e34c7ac2/0x73cfd86cfc17bb8c lrc: 4/2,0 mode: PR/PR res: [0x240000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051630 expref: -99 pid: 61474 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764723.813029:0:61474:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000e34c7ac2/0x73cfd86cfc17bb8c lrc: 4/2,0 mode: PR/PR res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051630 expref: -99 pid: 61474 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1775764723.813129:0:61474:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764723.813227:0:61474:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000e34c7ac2/0x73cfd86cfc17bb8c lrc: 7/2,0 mode: PR/PR res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051630 expref: -99 pid: 61474 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764723.813236:0:61474:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000e34c7ac2/0x73cfd86cfc17bb8c lrc: 6/1,0 mode: PR/PR res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051630 expref: -99 pid: 61474 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1775764723.813242:0:61474:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764723.813249:0:61474:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1:0x0] updating i_size 13 i_blocks 11 00000020:00010000:1.0:1775764723.813253:0:61474:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2389eaab58 (ffff8b238a3ff6c8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764723.813256:0:61474:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389eaab58 (ffff8b238a3ff6c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764723.813259:0:61474:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5130 (ffff8b2387cba908/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764723.813262:0:61474:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389eaab58 (ffff8b238a3ff6c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764723.813265:0:61474:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5130 (ffff8b238a3ff6c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764723.813275:0:61474:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:0 00000008:00010000:1.0:1775764723.814886:0:61474:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b238a0ce6c0 00000008:00010000:1.0:1775764723.814954:0:30878:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b238a0ce6c0 00010000:00010000:1.0:1775764723.814988:0:30878:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000e34c7ac2/0x73cfd86cfc17bb8c lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051630 expref: -99 pid: 61474 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00100000:1.0:1775764723.815034:0:30878:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23864fdf80 - object 0x240000400:280 offset 0<>4096 00000008:00010000:1.0:1775764723.825449:0:2377:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8b238a0ce6c0 00000020:00200000:1.0:1775764723.825511:0:2377:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1775764723.825521:0:2377:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8b238a0ce6c0 00000100:00100000:1.0:1775764723.825549:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098569344:192.168.202.128@tcp:3:lfs.0 00000100:00100000:1.0:1775764723.836990:0:61474:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61474:1862022098569600:192.168.202.128@tcp:35:lfs.0 00000002:00100000:1.0:1775764723.837027:0:61474:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea5500 x1862022098562432/t34359739053(34359739053) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764738 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764723.936992:0:61479:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764723.937018:0:61479:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764723.937036:0:61479:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764723.937050:0:61479:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764723.937056:0:61479:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764723.937062:0:61479:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764723.937065:0:61479:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bb9a 00000080:00010000:2.0:1775764723.937071:0:61479:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb9a from it ffffa60b01eabbf8 00010000:00010000:2.0:1775764723.937073:0:61479:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764723.937083:0:61479:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764723.937096:0:61479:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764723.937108:0:61479:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f22.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b2388002180, open_flags 8800, mode 0 opened 0 00000080:00200000:2.0:1775764723.937113:0:61479:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f22.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open 00000002:00010000:2.0:1775764723.937126:0:61479:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f22.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104001 00010000:00010000:2.0:1775764723.937191:0:61479:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 61479 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.937200:0:61479:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61479 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764723.937213:0:61479:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764723.937225:0:61479:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5f80 x1862022098570368/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764723.937232:0:61479:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61479 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764723.937254:0:61479:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61479:1862022098570368:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764723.939997:0:61479:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386f00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764723.950071:0:61479:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61479:1862022098570368:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764723.950097:0:61479:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.950114:0:61479:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.950123:0:61479:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.950134:0:61479:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.950146:0:61479:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.950159:0:61479:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.950167:0:61479:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.950174:0:61479:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764723.950184:0:61479:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23b9ea5f80 x1862022098570368/t34359739069(34359739069) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/864 e 0 to 0 dl 1775764739 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764723.950197:0:61479:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea5f80 x1862022098570368/t34359739069(34359739069) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/864 e 0 to 0 dl 1775764739 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764723.950210:0:61479:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764723.950224:0:61479:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.950234:0:61479:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/1 (0 0) 00000080:00200000:2.0:1775764723.950239:0:61479:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764723.950329:0:61479:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0x1:0x0](ffff8b2390ec21c8): rc = 0 00000080:00200000:2.0:1775764723.950349:0:61479:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1:0x0]: layout version change: 3 -> 3 00000080:00010000:2.0:1775764723.950360:0:61479:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) for lock 0x73cfd86cfc17bba8 00000080:00010000:2.0:1775764723.950373:0:61479:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) 00000080:00200008:2.0:1775764723.950402:0:61479:0:(file.c:1057:ll_file_open()) START file f22.sanity-pcc:[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), flags 104000 00000002:00100000:2.0:1775764723.950423:0:61479:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea5f80 x1862022098570368/t34359739069(34359739069) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764739 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764723.950508:0:61479:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bba8 from it ffff8b238504c120 00010000:00010000:2.0:1775764723.950513:0:61479:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764723.950523:0:61479:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764723.950554:0:61479:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), inode mode 81a4 mask 44 00000080:00200080:2.0:1775764723.950584:0:61479:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x1:0x0](ffff8b2390ec21c8) cmd=802066d3 arg=7ffded7eda10 00800000:00000080:2.0:1775764723.950596:0:61479:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=802066d3 len=384 karg=ffff8b238a3c2400 uarg=0000000000000000 00000002:00000080:2.0:1775764723.950603:0:61479:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=802066d3 len=384 karg=ffff8b238a3c2400 uarg=0000000000000000 00000100:00100000:2.0:1775764723.950654:0:61479:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61479:1862022098570496:192.168.202.128@tcp:54:lfs.0 00000100:00100000:2.0:1775764723.950742:0:61479:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386f00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764723.956753:0:61479:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61479:1862022098570496:192.168.202.128@tcp:54:lfs.0 00000080:00200008:2.0:1775764723.956817:0:61479:0:(file.c:447:ll_file_release()) START file f22.sanity-pcc:[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), flags 104000 00000002:00100000:2.0:1775764723.956882:0:61479:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea5f80 x1862022098570368/t34359739069(34359739069) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764739 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764723.956894:0:61479:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764723.956899:0:61479:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5c00 x1862022098570752/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764723.956923:0:61479:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61479:1862022098570752:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764723.957291:0:61479:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386f00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764723.961802:0:61479:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61479:1862022098570752:192.168.202.128@tcp:35:lfs.0 00000080:00200000:1.0:1775764724.203300:0:61500:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764724.203329:0:61500:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764724.203347:0:61500:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764724.203364:0:61500:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764724.203373:0:61500:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764724.203381:0:61500:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764724.203384:0:61500:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bb9a 00000080:00010000:1.0:1775764724.203391:0:61500:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb9a from it ffffa60b01ed3bf8 00010000:00010000:1.0:1775764724.203396:0:61500:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764724.203404:0:61500:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764724.203418:0:61500:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764724.203425:0:61500:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f22.sanity-pcc, flags=265 00000080:00200000:1.0:1775764724.203430:0:61500:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), inode mode 81a4 mask 44 00000080:00200008:1.0:1775764724.203436:0:61500:0:(file.c:1057:ll_file_open()) START file f22.sanity-pcc:[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), flags 100000 00000002:00010000:1.0:1775764724.203485:0:61500:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 04240100001 00010000:00010000:1.0:1775764724.203541:0:61500:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bbaf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 61500 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764724.203550:0:61500:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bbaf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61500 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764724.203562:0:61500:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764724.203577:0:61500:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fdf80 x1862022098571008/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764724.203587:0:61500:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bbaf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61500 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764724.203628:0:61500:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:61500:1862022098571008:192.168.202.128@tcp:101:cat.0 00000100:00100000:1.0:1775764724.204176:0:61500:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384904840 going to sleep for 11 seconds 00000080:00200000:1.0:1775764724.265377:0:61501:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764724.265419:0:61501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764724.265439:0:61501:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764724.265499:0:61501:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764724.265508:0:61501:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764724.265517:0:61501:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764724.265521:0:61501:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bb9a 00000080:00010000:1.0:1775764724.265529:0:61501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb9a from it ffffa60b01e83b58 00010000:00010000:1.0:1775764724.265533:0:61501:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764724.265541:0:61501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200000:1.0:1775764724.265558:0:61501:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764724.265565:0:61501:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f22.sanity-pcc, flags=8 00000080:00200008:1.0:1775764724.265573:0:61501:0:(file.c:6226:ll_getattr_dentry()) START file f22.sanity-pcc:[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764724.265577:0:61501:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x1:0x0](ffff8b2390ec21c8),name=f22.sanity-pcc 00000002:00010000:1.0:1775764724.265587:0:61501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764724.265594:0:61501:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764724.265612:0:61501:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764724.265619:0:61501:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764724.265623:0:61501:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) 00000080:00010000:1.0:1775764724.265626:0:61501:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) for lock 0x73cfd86cfc17bba8 00000080:00010000:1.0:1775764724.265629:0:61501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bba8 from it ffffa60b01e83cb8 00010000:00010000:1.0:1775764724.265632:0:61501:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764724.265638:0:61501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764724.265667:0:61501:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764724.265673:0:61501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764724.265679:0:61501:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764724.265687:0:61501:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200000:1.0:1775764724.266246:0:61501:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764724.266252:0:61501:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764724.266254:0:61501:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bb9a 00000080:00010000:1.0:1775764724.266259:0:61501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb9a from it ffffa60b01e83b98 00010000:00010000:1.0:1775764724.266262:0:61501:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764724.266269:0:61501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200000:1.0:1775764724.266278:0:61501:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764724.266283:0:61501:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f22.sanity-pcc, flags=8 00000080:00200008:1.0:1775764724.266349:0:61501:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764724.266353:0:61501:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764724.266361:0:61501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764724.266374:0:61501:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764724.266382:0:61501:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200000:1.0:1775764724.266389:0:61501:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764724.266392:0:61501:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764724.266394:0:61501:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bb9a 00000080:00010000:1.0:1775764724.266398:0:61501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb9a from it ffffa60b01e83cb8 00010000:00010000:1.0:1775764724.266400:0:61501:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764724.266407:0:61501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:1.0:1775764724.266421:0:61501:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764724.266426:0:61501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764724.266431:0:61501:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764724.266438:0:61501:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:1.0:1775764724.266443:0:61501:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764724.266446:0:61501:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764724.266449:0:61501:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bb9a 00000080:00010000:1.0:1775764724.266911:0:61501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb9a from it ffffa60b01e83b58 00010000:00010000:1.0:1775764724.266915:0:61501:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764724.266923:0:61501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:1.0:1775764724.266933:0:61501:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764724.266939:0:61501:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f22.sanity-pcc, flags=8 00000080:00200008:1.0:1775764724.266945:0:61501:0:(file.c:6226:ll_getattr_dentry()) START file f22.sanity-pcc:[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764724.266960:0:61501:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x1:0x0](ffff8b2390ec21c8),name=f22.sanity-pcc 00000002:00010000:1.0:1775764724.266972:0:61501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764724.266980:0:61501:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764724.266988:0:61501:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764724.267002:0:61501:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764724.267007:0:61501:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) 00000080:00010000:1.0:1775764724.267009:0:61501:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) for lock 0x73cfd86cfc17bba8 00000080:00010000:1.0:1775764724.267011:0:61501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bba8 from it ffffa60b01e83cb8 00010000:00010000:1.0:1775764724.267014:0:61501:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764724.267020:0:61501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200008:1.0:1775764724.268853:0:61501:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764724.268861:0:61501:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764724.268880:0:61501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764724.268895:0:61501:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764724.268909:0:61501:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:1.0:1775764724.268916:0:61501:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764724.268924:0:61501:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764724.268927:0:61501:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bb9a 00000080:00010000:1.0:1775764724.268934:0:61501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb9a from it ffffa60b01e83cb8 00010000:00010000:1.0:1775764724.268937:0:61501:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764724.269239:0:61501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:1.0:1775764724.269330:0:61501:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764724.269338:0:61501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764724.269346:0:61501:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:1.0:1775764724.269355:0:61501:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:1.0:1775764724.269366:0:61501:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764724.269371:0:61501:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764724.269374:0:61501:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bb9a 00000080:00010000:1.0:1775764724.269377:0:61501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb9a from it ffffa60b01e83bb0 00010000:00010000:1.0:1775764724.269380:0:61501:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:1.0:1775764724.269388:0:61501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:1.0:1775764724.269397:0:61501:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764724.269405:0:61501:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:1.0:1775764724.269808:0:61501:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=401866d9 arg=12a52a0 00000002:00010000:1.0:1775764724.269834:0:61501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1:0x0]) in obj [0x200003ab4:0x1:0x0], intent: open flags 03004040000002 00010000:00010000:1.0:1775764724.269849:0:61501:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bbaf lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e030516ae expref: -99 pid: 61500 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764724.269868:0:61501:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000002:00100000:1.0:1775764724.269915:0:61501:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fdf80 x1862022098571008/t34359739071(34359739071) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764740 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cat.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764724.269927:0:61501:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764724.269933:0:61501:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fed80 x1862022098571264/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764724.270422:0:61501:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61501:1862022098571264:192.168.202.128@tcp:35:lfs.0 00000100:00100000:1.0:1775764724.270876:0:61501:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849040c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764724.281303:0:61501:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61501:1862022098571264:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764724.291905:0:61501:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864f2680 x1862022098571520/t34359739073(34359739073) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764740 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764724.291915:0:61501:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) for lock 0x73cfd86cfc17bbb6 00000080:00010000:2.0:1775764724.291923:0:61501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bbb6 from it ffffa60b01e83b80 00010000:00010000:2.0:1775764724.291927:0:61501:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f800/0x73cfd86cfc17bbb6 lrc: 3/0,1 mode: PW/PW res: [0x200003ab4:0x1:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e030516c3 expref: -99 pid: 61501 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.291934:0:61501:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f800/0x73cfd86cfc17bbb6 lrc: 2/0,0 mode: PW/PW res: [0x200003ab4:0x1:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e030516c3 expref: -99 pid: 61501 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764724.292020:0:61501:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764724.292255:0:61501:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x1:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764724.292259:0:61501:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764724.292266:0:61501:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:1.0:1775764724.297256:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764724.300876:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764724.300885:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000e34c7ac2/0x73cfd86cfc17bb8c lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e03051630 expref: -99 pid: 61474 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 1 00010000:00010000:2.0:1775764724.300899:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5ea00 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:2.0:1775764724.300906:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000e34c7ac2/0x73cfd86cfc17bb8c lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800428400000000 nid: local remote: 0x57b8ff0e03051630 expref: -99 pid: 61474 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 1 00000080:00200000:2.0:1775764724.300965:0:46149:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764724.301066:0:46149:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775764724.301148:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000e34c7ac2/0x73cfd86cfc17bb8c lrc: 2/0,0 mode: --/PR res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e03051630 expref: -99 pid: 61474 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 1 00010000:00010000:2.0:1775764724.301157:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764724.301195:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000e34c7ac2/0x73cfd86cfc17bb8c lrc: 1/0,0 mode: --/PR res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e03051630 expref: -99 pid: 61474 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 1 00010000:00010000:2.0:1775764724.301203:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000e34c7ac2/0x73cfd86cfc17bb8c lrc: 0/0,0 mode: --/PR res: [0x240000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e03051630 expref: -99 pid: 61474 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 1 00010000:00010000:2.0:1775764724.301229:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764724.301230:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764724.301232:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764724.301245:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764724.301261:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098571776:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764724.312908:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098571776:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764724.313320:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098571648:192.168.202.128@tcp:1:lfs.0 00000080:00200000:1.0:1775764724.313848:0:61501:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1:0x0] updating i_size 13 i_blocks 11 00010000:00010000:1.0:1775764724.313909:0:61501:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f800/0x73cfd86cfc17bbb6 lrc: 2/0,0 mode: PW/PW res: [0x200003ab4:0x1:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x57b8ff0e030516c3 expref: -99 pid: 61501 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764724.313930:0:61501:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f800/0x73cfd86cfc17bbb6 lrc: 0/0,0 mode: --/PW res: [0x200003ab4:0x1:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x57b8ff0e030516c3 expref: -99 pid: 61501 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764724.314256:0:61501:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864f2680 x1862022098571520/t34359739073(34359739073) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764740 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764724.314269:0:61501:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764724.314274:0:61501:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fce00 x1862022098572032/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 504/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764724.314299:0:61501:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61501:1862022098572032:192.168.202.128@tcp:35:lfs.0 00000100:00100000:1.0:1775764724.314881:0:61501:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384904d80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764724.322422:0:46141:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119043840 00000100:00100000:2.0:1775764724.322439:0:46141:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119043840 00000100:00100000:2.0:1775764724.322561:0:46141:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 0 00000100:00100000:2.0:1775764724.322573:0:46141:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dae300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:47878:x1862022119043840:12345-192.168.202.128@tcp:104: 00010000:00010000:2.0:1775764724.322611:0:46141:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764724.322684:0:46141:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1775764724.322701:0:46141:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dae300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:47878:x1862022119043840:12345-192.168.202.128@tcp:104: Request processed in 120us (391us total) trans 0 rc 0/0 00000100:00100000:2.0:1775764724.322708:0:46141:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 0 00010000:00010000:2.0:1775764724.322731:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764724.322739:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764724.322747:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5fa00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764724.322752:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764724.322761:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764724.322769:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 3/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764724.322797:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764724.322832:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 2/0,0 mode: --/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764724.322838:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764724.322858:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764724.322866:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bb93 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e0305163e expref: -99 pid: 61474 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764724.322881:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764724.322883:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764724.322885:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764724.322899:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098572160:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764724.329283:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098572160:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764724.334507:0:46140:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764724.334544:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119043712 00000100:00100000:1.0:1775764724.334557:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119043712 00000100:00100000:1.0:1775764724.334651:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 218 00000100:00100000:1.0:1775764724.334663:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b239824a050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:47878:x1862022119043712:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764724.334682:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764724.334778:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764724.334785:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b239824a050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:47878:x1862022119043712:12345-192.168.202.128@tcp:104: Request processed in 125us (12598us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764724.334795:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 218 00010000:00010000:2.0:1775764724.335474:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764724.335484:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764724.335503:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6eaa00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764724.335508:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764724.335517:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764724.335525:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764724.335597:0:46149:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1:0x0]: losing layout lock 00010000:00010000:2.0:1775764724.335712:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764724.335720:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764724.335753:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764724.335761:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bba8 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03051692 expref: -99 pid: 61479 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764724.335780:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764724.335782:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764724.335784:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764724.335794:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764724.335813:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098572288:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764724.344922:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098572288:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764724.365341:0:61501:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61501:1862022098572032:192.168.202.128@tcp:35:lfs.0 00000002:00100000:1.0:1775764724.365381:0:61501:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864f2680 x1862022098571520/t34359739073(34359739073) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764740 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1775764724.373390:0:61501:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000100:00100000:2.0:1775764724.411255:0:61502:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61502:1862022098572544:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764724.411284:0:61502:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045000/0x73cfd86cfc17bbbd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.411299:0:61502:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045000/0x73cfd86cfc17bbbd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 61502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.411305:0:61502:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045000/0x73cfd86cfc17bbbd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 61502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.411313:0:61502:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045000/0x73cfd86cfc17bbbd lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 61502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.411320:0:61502:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045000/0x73cfd86cfc17bbbd lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 61502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.411326:0:61502:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23c2045000 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764724.411330:0:61502:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045000/0x73cfd86cfc17bbbd lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 61502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.411336:0:61502:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045000/0x73cfd86cfc17bbbd lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 61502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.411342:0:61502:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045000/0x73cfd86cfc17bbbd lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 61502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.411350:0:61502:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764724.411357:0:61502:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045000/0x73cfd86cfc17bbbd lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 61502 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.411362:0:61502:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045000/0x73cfd86cfc17bbbd lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 61502 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764724.411402:0:61502:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8b2382daf100 x1862022098572544/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/608 e 0 to 0 dl 1775764740 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764724.411418:0:61502:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764724.411785:0:61502:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764724.411804:0:61502:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764724.411819:0:61502:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764724.411831:0:61502:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:2.0:1775764724.411838:0:61502:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764724.411843:0:61502:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764724.411847:0:61502:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bb9a 00000080:00010000:2.0:1775764724.411852:0:61502:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb9a from it ffffa60b01efbbb0 00010000:00010000:2.0:1775764724.411855:0:61502:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764724.411863:0:61502:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:2.0:1775764724.411876:0:61502:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764724.411885:0:61502:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764724.411917:0:61502:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086915 arg=12fa3d0 00000100:00100000:2.0:1775764724.411973:0:61502:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61502:1862022098572672:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764724.412227:0:61502:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386f00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764724.421039:0:61502:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61502:1862022098572672:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764724.421293:0:61502:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:1.0:1775764724.639856:0:61515:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0x1:0x0], updating i_size 13 00000080:00200000:1.0:1775764724.639880:0:61515:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1:0x0]: layout version change: 4 -> 4 00000080:00010000:1.0:1775764724.639892:0:61515:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) for lock 0x73cfd86cfc17bbcb 00000080:00010000:1.0:1775764724.639900:0:61515:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bbcb from it ffffa60b01eebbf0 00010000:00010000:1.0:1775764724.639905:0:61515:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044200/0x73cfd86cfc17bbcb lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305173a expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764724.639917:0:61515:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044200/0x73cfd86cfc17bbcb lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305173a expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764724.639949:0:61515:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382dac000 x1862022098573568/t34359739084(34359739084) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764740 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764724.640041:0:61515:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x1:0x0](ffff8b2390ec21c8) cmd=400866fc arg=7ffd3cbad398 00800000:00000080:1.0:1775764724.640069:0:61515:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d9 len=56 karg=ffff8b23891edd80 uarg=0000000000000000 00000002:00000080:1.0:1775764724.640076:0:61515:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d9 len=56 karg=ffff8b23891edd80 uarg=0000000000000000 00010000:00010000:1.0:1775764724.640101:0:61515:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044200/0x73cfd86cfc17bbcb lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e0305173a expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764724.640126:0:61515:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1:0x0]: losing layout lock 00010000:00010000:1.0:1775764724.640146:0:61515:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbc4 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e0305171e expref: -99 pid: 61505 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764724.640168:0:61515:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1:0x0]: losing layout lock 00010000:00010000:1.0:1775764724.640191:0:61515:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044200/0x73cfd86cfc17bbcb lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e0305173a expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764724.640197:0:61515:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbc4 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e0305171e expref: -99 pid: 61505 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764724.640203:0:61515:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00000100:00100000:1.0:1775764724.640224:0:61515:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61515:1862022098573824:192.168.202.128@tcp:103:lfs.0 00000100:00100000:1.0:1775764724.640338:0:61515:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384904d80 going to sleep for 11 seconds 00010000:00010000:2.0:1775764724.663727:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764724.791887:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098574336:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764724.792030:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098574592:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764724.792070:0:2377:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1775764724.792078:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098574208:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764724.792110:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098574464:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764724.803255:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098574336:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764724.803271:0:2377:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b2389398000: committing for last_committed 34359738439 gen 1 00000100:00100000:1.0:1775764724.803290:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098574592:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764724.803309:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098574464:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764724.816662:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200080:1.0:1775764724.819569:0:61516:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086696 arg=7fa5a0000d20 00800000:00000080:1.0:1775764724.819593:0:61516:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=c0086696 len=4128 karg=ffff8b23ab52c000 uarg=0000000000000000 00000002:00000080:1.0:1775764724.819614:0:61516:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=c0086696 len=4128 karg=ffff8b23ab52c000 uarg=0000000000000000 00000002:00000080:1.0:1775764724.819621:0:61516:0:(mdc_request.c:1738:mdc_ioc_fid2path()) path get [0x200003ab4:0x1:0x0] from 18446744073709551615 #0 00000100:00100000:1.0:1775764724.819667:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098574720:192.168.202.128@tcp:53:lhsmtool_posix.0 00000100:00100000:1.0:1775764724.819870:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849040c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764724.832997:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098574720:192.168.202.128@tcp:53:lhsmtool_posix.0 00000002:00000080:1.0:1775764724.833035:0:61516:0:(mdc_request.c:1758:mdc_ioc_fid2path()) path got [0x200000007:0x1:0x0] from 18446744073709551615 #0: f22.sanity-pcc 00000080:00200080:1.0:1775764724.833191:0:61516:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c01066f8 arg=14749e8 00800000:00000080:1.0:1775764724.833202:0:61516:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=c01066f8 len=16 karg=ffffa60b01f13cf0 uarg=ffffa60b01f13cc0 00000080:00200080:1.0:1775764724.833365:0:61516:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000002:0x2:0x0](ffff8b2390ec5348) cmd=c1406916 arg=7fa5a0000d20 00000100:00100000:1.0:1775764724.836051:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098574848:192.168.202.128@tcp:34:lhsmtool_posix.0 00000100:00100000:1.0:1775764724.836137:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849040c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764724.842123:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098574848:192.168.202.128@tcp:34:lhsmtool_posix.0 00000080:00200000:1.0:1775764724.842227:0:61516:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000002:0x2:0x0](ffff8b2390ec5348), inode mode 4040 mask 1 00000080:00200000:1.0:1775764724.842253:0:61516:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=0x200003ab4:0x1:0x0, dir=[0x200000002:0x2:0x0](ffff8b2390ec5348), file ffff8b23a0ec3200, open_flags 68800, mode 0 opened 0 00000080:00200000:1.0:1775764724.842262:0:61516:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=0x200003ab4:0x1:0x0, dir=[0x200000002:0x2:0x0](ffff8b2390ec5348), intent=open 00000002:00010000:1.0:1775764724.842293:0:61516:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1:0x0]) in obj [0x200000002:0x2:0x0], intent: open flags 041504001 00010000:00010000:1.0:1775764724.842367:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bbd9 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764724.842378:0:61516:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bbd9 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764724.842390:0:61516:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764724.844130:0:61516:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ffb80 x1862022098575104/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764724.844151:0:61516:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bbd9 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764724.844192:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098575104:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764724.844495:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849040c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764724.848296:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098575104:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:1.0:1775764724.848319:0:61516:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bbd9 lrc: 4/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764724.848332:0:61516:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bbd9 lrc: 4/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764724.848339:0:61516:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bbd9 lrc: 4/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764724.848348:0:61516:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bbd9 lrc: 3/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764724.848357:0:61516:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bbd9 lrc: 4/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764724.848364:0:61516:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8740800 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764724.848368:0:61516:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bbd9 lrc: 4/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764724.848375:0:61516:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bbd9 lrc: 4/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764724.848383:0:61516:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bbd9 lrc: 5/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764724.848391:0:61516:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764724.848412:0:61516:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bbd9 lrc: 3/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764724.848418:0:61516:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bbd9 lrc: 0/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764724.848438:0:61516:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23864ffb80 x1862022098575104/t34359739085(34359739085) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764740 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764724.848450:0:61516:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864ffb80 x1862022098575104/t34359739085(34359739085) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764740 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764724.850692:0:61516:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000002:0x2:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764724.854752:0:61516:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), inode mode 81a4 mask 44 00000080:00200080:2.0:1775764724.854794:0:61516:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086696 arg=7fa5a0000d20 00800000:00000080:2.0:1775764724.854815:0:61516:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=c0086696 len=4128 karg=ffff8b2384974000 uarg=0000000000000000 00000002:00000080:2.0:1775764724.854822:0:61516:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=c0086696 len=4128 karg=ffff8b2384974000 uarg=0000000000000000 00000002:00000080:2.0:1775764724.854829:0:61516:0:(mdc_request.c:1738:mdc_ioc_fid2path()) path get [0x200003ab4:0x1:0x0] from 18446744073709551615 #0 00000100:00100000:2.0:1775764724.854872:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098575488:192.168.202.128@tcp:53:lhsmtool_posix.0 00000100:00100000:2.0:1775764724.854947:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386f00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764724.859208:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098575488:192.168.202.128@tcp:53:lhsmtool_posix.0 00000002:00000080:2.0:1775764724.859237:0:61516:0:(mdc_request.c:1758:mdc_ioc_fid2path()) path got [0x200000007:0x1:0x0] from 18446744073709551615 #0: f22.sanity-pcc 00000080:00200000:2.0:1775764724.859315:0:61516:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764724.859338:0:61516:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764724.859363:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:2.0:1775764724.859377:0:61516:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:2.0:1775764724.859386:0:61516:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764724.859393:0:61516:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764724.859794:0:61516:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bb9a 00000080:00010000:2.0:1775764724.859804:0:61516:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb9a from it ffffa60b01f13bf8 00010000:00010000:2.0:1775764724.859808:0:61516:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:2.0:1775764724.859819:0:61516:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:2.0:1775764724.859835:0:61516:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764724.859856:0:61516:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764724.859864:0:61516:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764724.859872:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:2.0:1775764724.859880:0:61516:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:2.0:1775764724.859887:0:61516:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764724.859891:0:61516:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764724.859893:0:61516:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bb9a 00000080:00010000:2.0:1775764724.859897:0:61516:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bb9a from it ffffa60b01f13b38 00010000:00010000:2.0:1775764724.859900:0:61516:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:2.0:1775764724.859907:0:61516:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:2.0:1775764724.859915:0:61516:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:2.0:1775764724.859920:0:61516:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=. :VOLATILE:0000:6F3D6CF2:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b23903e9800, open_flags 280c2, mode 8180 opened 0 00000080:00200000:2.0:1775764724.859927:0:61516:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=. :VOLATILE:0000:6F3D6CF2:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000002:00010000:2.0:1775764724.859949:0:61516:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: . :VOLATILE:0000:6F3D6CF2:fd=09,[0x200003ab4:0x5:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0500303 00010000:00010000:2.0:1775764724.859961:0:61516:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:2.0:1775764724.860007:0:61516:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:2.0:1775764724.860015:0:61516:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764724.860019:0:61516:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bb9a lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305164c expref: -99 pid: 61476 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:2.0:1775764724.860056:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.860063:0:61516:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764724.860072:0:61516:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764724.860081:0:61516:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7100 x1862022098575744/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1088/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764724.860090:0:61516:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764724.860111:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098575744:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764724.860249:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386f00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764724.875928:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098575744:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764724.875961:0:61516:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051787 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.875997:0:61516:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051787 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.876005:0:61516:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051787 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.876020:0:61516:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051787 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.876032:0:61516:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051787 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.876126:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051787 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.876137:0:61516:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051787 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.876144:0:61516:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051787 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764724.876238:0:61516:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8b23b9ea7100 x1862022098575744/t34359739087(34359739087) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1088/664 e 0 to 0 dl 1775764740 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764724.876252:0:61516:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7100 x1862022098575744/t34359739087(34359739087) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1088/664 e 0 to 0 dl 1775764740 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764724.876267:0:61516:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051787 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764724.876282:0:61516:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051787 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.876293:0:61516:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/5 (0 0) 00000080:00200000:2.0:1775764724.876300:0:61516:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764724.876315:0:61516:0:(namei.c:1620:ll_create_it()) VFS Op:name=. :VOLATILE:0000:6F3D6CF2:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000080:00200000:2.0:1775764724.876345:0:61516:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab4:0x5:0x0](ffff8b2390ec42c8) 00000080:00200000:2.0:1775764724.876871:0:61516:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x5:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1775764724.876883:0:61516:0:(namei.c:1586:ll_create_node()) setting l_ast_data to inode [0x200003ab4:0x5:0x0](ffff8b2390ec42c8) 00000080:00010000:2.0:1775764724.876887:0:61516:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x5:0x0](ffff8b2390ec42c8) for lock 0x73cfd86cfc17bbe0 00000080:00010000:2.0:1775764724.876896:0:61516:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x5:0x0](ffff8b2390ec42c8) 00000080:00200008:2.0:1775764724.876910:0:61516:0:(file.c:1057:ll_file_open()) START file . :VOLATILE:0000:6F3D6CF2:fd=09:[0x200003ab4:0x5:0x0](ffff8b2390ec42c8), flags 500302 00000002:00100000:2.0:1775764724.876932:0:61516:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7100 x1862022098575744/t34359739087(34359739087) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 608/664 e 0 to 0 dl 1775764740 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764724.876974:0:61516:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bbe0 from it ffff8b238504c7e0 00010000:00010000:2.0:1775764724.876979:0:61516:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051787 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.876990:0:61516:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051787 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764724.877030:0:61516:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x5:0x0](ffff8b2390ec42c8), inode mode 8180 mask 40 00000080:00200000:2.0:1775764724.878267:0:61516:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764724.878292:0:61516:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764724.878363:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.878372:0:61516:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.878382:0:61516:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764724.878409:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098575872:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764724.879847:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764724.885388:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098575872:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764724.885432:0:61516:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305179c expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.885445:0:61516:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305179c expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.885859:0:61516:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305179c expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.885873:0:61516:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305179c expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.885888:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305179c expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.885897:0:61516:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305179c expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.885904:0:61516:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305179c expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764724.885917:0:61516:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea5c00 x1862022098575872/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764740 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764724.885931:0:61516:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305179c expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.885952:0:61516:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764724.885959:0:61516:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764724.885992:0:61516:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764724.885998:0:61516:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bbe7 00000080:00010000:2.0:1775764724.886004:0:61516:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764724.886007:0:61516:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bbe7 from it ffffa60b01f13c10 00010000:00010000:2.0:1775764724.886010:0:61516:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305179c expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.886019:0:61516:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305179c expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764724.886044:0:61516:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764724.886063:0:61516:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=. :VOLATILE:0000:6F3D6CF2:fd=09, flags=0 00000080:00200000:2.0:1775764724.886070:0:61516:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764724.886086:0:61516:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764724.886095:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305179c expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764724.886106:0:61516:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305179c expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764724.886112:0:61516:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764724.886116:0:61516:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764724.886119:0:61516:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bbe7 00000080:00010000:2.0:1775764724.886122:0:61516:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bbe7 from it ffffa60b01f13d70 00010000:00010000:2.0:1775764724.886125:0:61516:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305179c expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764724.886131:0:61516:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305179c expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764724.886138:0:61516:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:2.0:1775764724.886143:0:61516:0:(namei.c:2405:ll_unlink()) VFS Op:name=. :VOLATILE:0000:6F3D6CF2:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08) 00010000:00010000:2.0:1775764724.886219:0:61516:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0305179c expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764724.886247:0:61516:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03051787 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764724.886266:0:61516:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x5:0x0]: losing layout lock 00010000:00010000:2.0:1775764724.886306:0:61516:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051787 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764724.886314:0:61516:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305179c expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764724.886319:0:61516:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1775764724.886321:0:61516:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbe0 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051787 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764724.886339:0:61516:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbe7 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305179c expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000020:00100000:2.0:1775764724.886356:0:61516:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764724.886364:0:61516:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5c00 x1862022098576128/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/1032 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764724.886386:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098576128:192.168.202.128@tcp:36:lhsmtool_posix.0 00000100:00100000:2.0:1775764724.886893:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764724.892175:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098576128:192.168.202.128@tcp:36:lhsmtool_posix.0 00000002:00010000:2.0:1775764724.892251:0:61516:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x0:0x0:0x0]) in obj [0x200003ab4:0x5:0x0], intent: getxattr flags 0 00010000:00010000:2.0:1775764724.892307:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bbee lrc: 3/1,0 mode: --/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.892317:0:61516:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bbee lrc: 3/1,0 mode: --/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.892328:0:61516:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bbee lrc: 3/1,0 mode: --/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764724.892348:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098576256:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764724.892433:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764724.898656:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098576256:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764724.898688:0:61516:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bbee lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030517b8 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.898704:0:61516:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bbee lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030517b8 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.898716:0:61516:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bbee lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030517b8 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764724.898735:0:61516:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=80 disposition=0, status=0 req@ffff8b23b9ea5c00 x1862022098576256/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 568/1072 e 0 to 0 dl 1775764740 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764724.898746:0:61516:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bbee lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030517b8 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.898759:0:61516:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/5 (0 0) 00000080:00200000:2.0:1775764724.898765:0:61516:0:(namei.c:953:ll_intent_lock()) intent lock 128 on i1 [0x200003ab4:0x5:0x0] suppgids 0 -1: rc 0 00000080:00010000:2.0:1775764724.898789:0:61516:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x5:0x0](ffff8b2390ec42c8) for lock 0x73cfd86cfc17bbee 00000080:00010000:2.0:1775764724.898798:0:61516:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bbee from it ffffa60b01f13c28 00010000:00010000:2.0:1775764724.898802:0:61516:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bbee lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030517b8 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.898811:0:61516:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bbee lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030517b8 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1775764724.898849:0:61516:0:(llite_lib.c:2274:ll_setattr_raw()) START file . :VOLATILE:0000:6F3D6CF2:fd=09:[0x200003ab4:0x5:0x0](ffff8b2390ec42c8) current size 0, valid attrs 1846, mode 8b23, uid 0, gid 0, new size 0, atime 1775764724.000000000, mtime 1775764724.000000000, ctime 1775764724.000000000 00000020:00100000:2.0:1775764724.898894:0:61516:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764724.898901:0:61516:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5c00 x1862022098576512/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764724.898923:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098576512:192.168.202.128@tcp:36:lhsmtool_posix.0 00000100:00100000:2.0:1775764724.898995:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764724.904681:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098576512:192.168.202.128@tcp:36:lhsmtool_posix.0 00010000:00010000:2.0:1775764724.904743:0:61516:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 31 res 8589949620/5 (0 0) 00000080:00010000:2.0:1775764724.904757:0:61516:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x5:0x0](ffff8b2390ec42c8) 00000002:00010000:2.0:1775764724.904766:0:61516:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x5:0x0]) in obj [0x200003ab4:0x5:0x0], intent: layout flags 0 00010000:00010000:2.0:1775764724.904828:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.904838:0:61516:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.904848:0:61516:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764724.904870:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098576640:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764724.904928:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764724.909611:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098576640:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764724.909642:0:61516:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.909656:0:61516:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764724.909668:0:61516:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea5f80 x1862022098576640/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/400 e 0 to 0 dl 1775764740 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764724.909676:0:61516:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764724.909685:0:61516:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.909698:0:61516:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/5 (0 0) 00000080:00200000:2.0:1775764724.909703:0:61516:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x5:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764724.909716:0:61516:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x5:0x0](ffff8b2390ec42c8) for lock 0x73cfd86cfc17bbf5 00000080:00010000:2.0:1775764724.909723:0:61516:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bbf5 from it ffffa60b01f13a68 00010000:00010000:2.0:1775764724.909725:0:61516:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.909733:0:61516:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764724.909749:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764724.909756:0:61516:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764724.909761:0:61516:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x5:0x0](00000000754242ec) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764724.909808:0:61516:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x5:0x0]: layout version change: 4294967294 -> 0 00010000:00010000:2.0:1775764724.909824:0:61516:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764724.909828:0:61516:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764724.909849:0:61516:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x5:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764724.909860:0:61516:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764724.914024:0:61516:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x5:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1775764724.914040:0:61516:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file . :VOLATILE:0000:6F3D6CF2:fd=09:[0x200003ab4:0x5:0x0](ffff8b2390ec42c8) current size 0, valid attrs 1846, mode 8b23, uid 0, gid 0, new size 0, atime 1775764724.000000000, mtime 1775764724.000000000, ctime 1775764724.000000000, rc 0 00000080:00200008:2.0:1775764724.914091:0:61516:0:(file.c:447:ll_file_release()) START file 0x200003ab4:0x1:0x0:[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), flags 1504000 00000080:00200080:2.0:1775764724.914130:0:61516:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=400866d6 arg=7fa5a0000b78 00800000:00000080:2.0:1775764724.914137:0:61516:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b01f13cf0 uarg=0000000000000000 00000002:00000080:2.0:1775764724.914142:0:61516:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b01f13cf0 uarg=0000000000000000 00000020:00100000:2.0:1775764724.914166:0:61516:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 57, max 7 00000100:00100000:2.0:1775764724.914173:0:61516:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5500 x1862022098577024/t0(0) o57->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764724.914195:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098577024:192.168.202.128@tcp:57:lhsmtool_posix.0 00000100:00100000:2.0:1775764724.914331:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764724.920378:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098577024:192.168.202.128@tcp:57:lhsmtool_posix.0 00000080:00200008:2.0:1775764724.920700:0:61516:0:(file.c:6226:ll_getattr_dentry()) START file . :VOLATILE:0000:6F3D6CF2:fd=09:[0x200003ab4:0x5:0x0](ffff8b2390ec42c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764724.920708:0:61516:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x5:0x0](ffff8b2390ec42c8),name=. :VOLATILE:0000:6F3D6CF2:fd=09 00000002:00010000:2.0:1775764724.920726:0:61516:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x5:0x0]) in obj [0x200003ab4:0x5:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764724.920740:0:61516:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 30 res 8589949620/5 (0 0) 00010000:00010000:2.0:1775764724.920780:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbfc lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.920787:0:61516:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbfc lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.920794:0:61516:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbfc lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 61516 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764724.920810:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098577152:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764724.920868:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764724.925200:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098577152:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764724.925228:0:61516:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbfc lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030517d4 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.925238:0:61516:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbfc lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030517d4 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.925247:0:61516:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbfc lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030517d4 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.925255:0:61516:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbfc lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030517d4 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.925264:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbfc lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030517d4 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.925270:0:61516:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbfc lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030517d4 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.925275:0:61516:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbfc lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030517d4 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764724.925282:0:61516:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea5500 x1862022098577152/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/640 e 0 to 0 dl 1775764740 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764724.925297:0:61516:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbfc lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030517d4 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764724.925312:0:61516:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbfc lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030517d4 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.925325:0:61516:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/5 (0 0) 00000080:00200000:2.0:1775764724.925332:0:61516:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200003ab4:0x5:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764724.925439:0:61516:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x5:0x0]: layout version change: 0 -> 0 00000080:00010000:2.0:1775764724.925802:0:61516:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x5:0x0](ffff8b2390ec42c8) for lock 0x73cfd86cfc17bbfc 00000080:00010000:2.0:1775764724.925809:0:61516:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x5:0x0](ffff8b2390ec42c8) 00000080:00010000:2.0:1775764724.925812:0:61516:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bbfc from it ffffa60b01f13cd8 00010000:00010000:2.0:1775764724.925814:0:61516:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbfc lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030517d4 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.925821:0:61516:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbfc lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030517d4 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764724.925870:0:61516:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x5:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1775764724.925880:0:61516:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x5:0x0] 00000008:00010000:2.0:1775764724.925905:0:61516:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764724.925907:0:61516:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24cb0, osc lock ffff8b238a3f8d10, flags 0x400001002 00010000:00010000:2.0:1775764724.925958:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bc03 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.925969:0:61516:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bc03 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1775764724.926322:0:61516:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1775764724.928842:0:61516:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x5:0x0] updating i_size 0 i_blocks 1 00000020:00010000:2.0:1775764724.928848:0:61516:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027e6c8 (ffff8b23812ecb48/0) at cl_lock_release():245 00000020:00010000:2.0:1775764724.928851:0:61516:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027e6c8 (ffff8b23812ecb48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764724.928856:0:61516:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24cb0 (ffff8b23812ec488/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764724.928860:0:61516:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027e6c8 (ffff8b23812ecb48/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764724.928863:0:61516:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24cb0 (ffff8b23812ecb48/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764724.928905:0:61516:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x5:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:2.0:1775764724.928937:0:61516:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=401866d8 arg=7fa5a7a76910 00800000:00000080:2.0:1775764724.928946:0:61516:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b01f13cf0 uarg=0000000000000000 00000002:00000080:2.0:1775764724.928956:0:61516:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b01f13cf0 uarg=0000000000000000 00000020:00100000:2.0:1775764724.929147:0:61516:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 57, max 7 00000100:00100000:2.0:1775764724.929215:0:61516:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4380 x1862022098577536/t0(0) o57->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764724.929241:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098577536:192.168.202.128@tcp:57:lhsmtool_posix.0 00000100:00100000:2.0:1775764724.929356:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764724.933828:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098577536:192.168.202.128@tcp:57:lhsmtool_posix.0 00000080:00200008:2.0:1775764724.934015:0:61516:0:(file.c:2612:do_file_write_iter()) START file . :VOLATILE:0000:6F3D6CF2:fd=09:[0x200003ab4:0x5:0x0], ppos: 0, count: 13 00000080:00200000:2.0:1775764724.934035:0:61516:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 13 bytes 00000080:00200000:2.0:1775764724.934040:0:61516:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 13 00000080:00200000:2.0:1775764724.934076:0:61516:0:(file.c:2058:ll_file_io_generic()) . :VOLATILE:0000:6F3D6CF2:fd=09: write ppos: 0, bytes: 13 00000080:00200000:2.0:1775764724.934081:0:61516:0:(file.c:1815:ll_io_set_mirror()) . :VOLATILE:0000:6F3D6CF2:fd=09: desiginated mirror: 0 00000020:00200000:2.0:1775764724.934089:0:61516:0:(cl_io.c:214:cl_io_rw_init()) header@00000000e41537d9[0x110, 1, [0x200003ab4:0x5:0x0] hash] 00000020:00200000:2.0:1775764724.934091:0:61516:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 13) 0 0 00000080:00200000:2.0:1775764724.934096:0:61516:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x5:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764724.934111:0:61516:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:2.0:1775764724.934118:0:61516:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 13) 13, 13 00020000:00200000:2.0:1775764724.934121:0:61516:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764724.934149:0:61516:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1775764724.934163:0:61516:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764724.934168:0:61516:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24cb0, osc lock ffff8b238a3f8d10, flags 0x0 00010000:00010000:2.0:1775764724.934186:0:61516:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2386938000 type 11 mode 2 res 10737419264/280 (0 4095) 00010000:00010000:2.0:1775764724.934229:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 3/0,1 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.934239:0:61516:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 3/0,1 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.934251:0:61516:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 3/0,1 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764724.934270:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098577664:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764724.934590:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764724.938150:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764724.938568:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764724.938576:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bc03 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e030517db expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764724.938595:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6eb200 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:2.0:1775764724.938614:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bc03 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x57b8ff0e030517db expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764724.938675:0:46149:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764724.938689:0:46149:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775764724.938750:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bc03 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e030517db expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764724.938758:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764724.938784:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bc03 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e030517db expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764724.938792:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bc03 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e030517db expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764724.938810:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764724.938813:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764724.938814:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764724.941941:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098577664:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764724.941978:0:61516:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 4/0,1 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.941993:0:61516:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 4/0,1 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.942002:0:61516:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 4/0,1 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764724.943211:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764724.943805:0:61516:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.943816:0:61516:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.943834:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1775764724.943846:0:61516:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.943860:0:61516:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.943869:0:61516:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764724.943902:0:61516:0:(vvp_io.c:1286:vvp_io_write_start()) . :VOLATILE:0000:6F3D6CF2:fd=09: write [0, 13) append 0 append lockpos 0 00000080:00200000:2.0:1775764724.943923:0:61516:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 13 bytes 00000080:00200000:2.0:1775764724.943991:0:61516:0:(rw26.c:858:ll_write_end()) pos 0, len 13, copied 13 00000080:00200000:2.0:1775764724.943999:0:61516:0:(rw26.c:916:ll_write_end()) page@00000000d1ce3a9a[3 00000000178597bf 2 1 0000000084fe28b7] 00000080:00200000:2.0:1775764724.944002:0:61516:0:(rw26.c:916:ll_write_end()) vmpage @0000000032388014 17ffffc0001001 3:0 ffff8b23baeaec40 0 lru 00000080:00200000:2.0:1775764724.944010:0:61516:0:(rw26.c:916:ll_write_end()) osc-page@00000000c66b2839 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 000000008819327c 000000005cb529b8 > 3< 0 0 > 4< 0 0 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764724.944012:0:61516:0:(rw26.c:916:ll_write_end()) end page@00000000d1ce3a9a 00000080:00200000:2.0:1775764724.944014:0:61516:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764724.944019:0:61516:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 13 prio 0 00000008:00010000:2.0:1775764724.944040:0:61516:0:(osc_cache.c:848:osc_extent_find()) ### extent: 00000000351313f8 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764724.944059:0:61516:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2390ec4478, count 1, dirtied 1 00000080:00200000:2.0:1775764724.944063:0:61516:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 13 bytes, tot: 13 00000080:00200000:2.0:1775764724.944067:0:61516:0:(vvp_io.c:1353:vvp_io_write_start()) . :VOLATILE:0000:6F3D6CF2:fd=09: write bytes 0, result: 13 00000020:00010000:2.0:1775764724.944083:0:61516:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b4e8 (ffff8b23812ecb48/0) at cl_lock_release():245 00000020:00010000:2.0:1775764724.944086:0:61516:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b4e8 (ffff8b23812ecb48/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764724.944090:0:61516:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24cb0 (ffff8b23812ec488/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764724.944094:0:61516:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764724.944103:0:61516:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1775764724.944117:0:61516:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b4e8 (ffff8b23812ecb48/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764724.944120:0:61516:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24cb0 (ffff8b23812ecb48/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764724.944128:0:61516:0:(vvp_io.c:517:vvp_io_advance()) advancing 13 bytes 00000080:00200000:2.0:1775764724.944136:0:61516:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:2.0:1775764724.944174:0:61516:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x5:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764724.944178:0:61516:0:(file.c:2206:ll_file_io_generic()) . :VOLATILE:0000:6F3D6CF2:fd=09: 2 io complete with rc: 0, result: 13, restart: 0 00000080:00200000:2.0:1775764724.944185:0:61516:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x5:0x0] updating i_size 13 i_blocks 1 00000080:00200000:2.0:1775764724.944189:0:61516:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 13 00000080:00200000:2.0:1775764724.944331:0:61516:0:(file.c:5304:ll_fsync()) VFS Op:inode=[0x200003ab4:0x5:0x0](ffff8b2390ec42c8), start 0, end 9223372036854775807, datasync 0 00000080:00200000:2.0:1775764724.944359:0:61516:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764724.944368:0:61516:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:2.0:1775764724.944394:0:61516:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 00000000351313f8 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1775764724.944407:0:61516:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000351313f8 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764724.944481:0:61516:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00100000:2.0:1775764724.944569:0:61516:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea4380 - object 0x280000400:280 offset 0<>13 00000080:00200000:2.0:1775764724.948265:0:61516:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:2.0:1775764724.948347:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098578048:192.168.202.128@tcp:44:lhsmtool_posix.0 00000100:00100000:2.0:1775764724.948415:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764725.007188:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098578048:192.168.202.128@tcp:44:lhsmtool_posix.0 00000080:00200000:2.0:1775764725.007258:0:61516:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x5:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764725.007270:0:61516:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764725.076630:0:61516:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x5:0x0] updating i_size 13 i_blocks 11 00000080:00200000:2.0:1775764725.076681:0:61516:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x5:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:2.0:1775764725.076707:0:61516:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x5:0x0](ffff8b2390ec42c8) cmd=c03066b0 arg=7fa5a7a75ac0 00000080:00200008:2.0:1775764725.076717:0:61516:0:(llite_lib.c:2274:ll_setattr_raw()) START file . :VOLATILE:0000:6F3D6CF2:fd=09:[0x200003ab4:0x5:0x0](ffff8b2390ec42c8) current size 13, valid attrs 1f0, mode 0, uid 0, gid 0, new size 0, atime 1775764724.000000000, mtime 1775764721.000000000, ctime 1775764723.000000000 00010000:00010000:2.0:1775764725.076750:0:61516:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbfc lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e030517d4 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.076772:0:61516:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764725.076816:0:61516:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbfc lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030517d4 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.076821:0:61516:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764725.076823:0:61516:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bbfc lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030517d4 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000020:00100000:2.0:1775764725.076843:0:61516:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764725.076853:0:61516:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4700 x1862022098578432/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764725.076875:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098578432:192.168.202.128@tcp:36:lhsmtool_posix.0 00000100:00100000:2.0:1775764725.076957:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764725.081631:0:61516:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 34359739088 gen 1 00000100:00100000:2.0:1775764725.081686:0:61516:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b2382dac000 x1862022098573568/t34359739084(34359739084) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764740 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764725.081698:0:61516:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23b9ea7100 x1862022098575744/t34359739087(34359739087) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 608/664 e 0 to 0 dl 1775764740 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764725.081708:0:61516:0:(client.c:3126:ptlrpc_free_committed()) @@@ stopping search req@ffff8b23b9ea4700 x1862022098578432/t34359739089(34359739089) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 488/456 e 0 to 0 dl 1775764741 ref 3 fl Rpc:RQU/204/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764725.081722:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098578432:192.168.202.128@tcp:36:lhsmtool_posix.0 00000080:00200000:2.0:1775764725.081833:0:61516:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x5:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764725.081846:0:61516:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764725.081872:0:61516:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:2.0:1775764725.081889:0:61516:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764725.081894:0:61516:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24cb0, osc lock ffff8b238a3f8d10, flags 0x0 00010000:00010000:2.0:1775764725.081920:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.081939:0:61516:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.081960:0:61516:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.081969:0:61516:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.081978:0:61516:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000100:00100000:1.0:1775764725.090953:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764725.091620:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764725.091629:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bbee lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e030517b8 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.091638:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6ebc00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764725.091644:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bbee lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e030517b8 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.091653:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bbee lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e030517b8 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.091662:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bbee lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x57b8ff0e030517b8 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.091720:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bbee lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030517b8 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.091733:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764725.091776:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bbee lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030517b8 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.091783:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bbee lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x5:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030517b8 expref: -99 pid: 61516 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.091803:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764725.091805:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764725.091806:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764725.091815:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764725.091829:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098578944:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764725.098318:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098578944:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764725.101398:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098579072:192.168.202.128@tcp:1:lhsmtool_posix.0 00000100:00100000:1.0:1775764725.104772:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764725.107140:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764725.107225:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800420000020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 1 00010000:00010000:2.0:1775764725.107240:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6eb800 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:2.0:1775764725.107247:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800428400020000 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 1 00000080:00200000:2.0:1775764725.107312:0:46149:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764725.107624:0:46149:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775764725.107710:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 2/0,0 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 1 00010000:00010000:2.0:1775764725.107724:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764725.108298:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 1/0,0 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 1 00010000:00010000:2.0:1775764725.108309:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc0a lrc: 0/0,0 mode: --/PW res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e030517e2 expref: -99 pid: 61516 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 1 00010000:00010000:2.0:1775764725.108338:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764725.108340:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764725.108343:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00800000:00000080:2.0:1775764725.112850:0:61516:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b01f13cf0 uarg=0000000000000000 00000002:00000080:2.0:1775764725.112864:0:61516:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b01f13cf0 uarg=0000000000000000 00000020:00100000:2.0:1775764725.112897:0:61516:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 57, max 7 00000100:00100000:2.0:1775764725.112905:0:61516:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4000 x1862022098579456/t0(0) o57->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764725.112932:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098579456:192.168.202.128@tcp:57:lhsmtool_posix.0 00000100:00100000:2.0:1775764725.113095:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764725.117796:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764725.117840:0:46140:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764725.117860:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119045120 00000100:00100000:1.0:1775764725.117870:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119045120 00000100:00100000:1.0:1775764725.117929:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 224 00000100:00100000:1.0:1775764725.117952:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b238693a450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119045120:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764725.117974:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764725.118060:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764725.118067:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b238693a450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119045120:12345-192.168.202.128@tcp:104: Request processed in 129us (285us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764725.118074:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 224 00010000:00010000:2.0:1775764725.118542:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764725.118549:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764725.118558:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6ea000 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764725.118562:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764725.118576:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764725.118586:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 3/0,0 mode: CR/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764725.118643:0:46150:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x5:0x0]: losing layout lock 00010000:00010000:2.0:1775764725.118700:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 2/0,0 mode: --/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764725.118707:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764725.118730:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764725.118738:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bbf5 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x5:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e030517c6 expref: -99 pid: 61516 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764725.118758:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764725.118759:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764725.118761:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764725.129222:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764725.129851:0:61515:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.129862:0:61515:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764725.129882:0:61515:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23864f3800 x1862022098574080/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/344 e 0 to 0 dl 1775764740 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764725.129901:0:61515:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.129917:0:61515:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/1 (0 0) 00000080:00200000:2.0:1775764725.129924:0:61515:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x1:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764725.129951:0:61515:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) for lock 0x73cfd86cfc17bbd2 00000080:00010000:2.0:1775764725.129961:0:61515:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bbd2 from it ffffa60b01eebbe8 00010000:00010000:2.0:1775764725.129966:0:61515:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.129977:0:61515:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.130005:0:61515:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.130018:0:61515:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764725.130026:0:61515:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1:0x0](00000000492670d3) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:2.0:1775764725.130084:0:61515:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61515:1862022098579712:192.168.202.128@tcp:49:lfs.0 00000100:00100000:2.0:1775764725.130149:0:61515:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764725.134836:0:61515:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61515:1862022098579712:192.168.202.128@tcp:49:lfs.0 00010000:00010000:2.0:1775764725.134971:0:61515:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.134989:0:61515:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764725.135031:0:61515:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764725.135040:0:61515:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764725.135077:0:61515:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.135089:0:61515:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764725.135098:0:61515:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1:0x0](00000000492670d3) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764725.135136:0:61515:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764725.135140:0:61515:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764725.135203:0:61515:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1:0x0]: layout version change: 4294967294 -> 5 00010000:00010000:2.0:1775764725.135212:0:61515:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.135220:0:61515:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00010000:2.0:1775764725.135251:0:61515:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) 00000002:00010000:2.0:1775764725.135261:0:61515:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1:0x0]) in obj [0x200003ab4:0x1:0x0], intent: layout flags 02 00010000:00010000:2.0:1775764725.135276:0:61515:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764725.135295:0:61515:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1:0x0]: losing layout lock 00010000:00010000:2.0:1775764725.135330:0:61515:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764725.135337:0:61515:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764725.135339:0:61515:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bbd2 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e0305174f expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764725.135390:0:61515:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 61515 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.135397:0:61515:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61515 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764725.135409:0:61515:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 2 is allocated opc 101, max 7 00000100:00100000:2.0:1775764725.135418:0:61515:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7b80 x1862022098579968/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/880 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764725.135427:0:61515:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61515 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764725.135449:0:61515:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61515:1862022098579968:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764725.135797:0:61515:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764725.149536:0:61515:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61515:1862022098579968:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764725.149559:0:61515:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 4/0,1 mode: --/EX res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051821 expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.149569:0:61515:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 4/0,1 mode: --/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051821 expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.149580:0:61515:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051821 expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.149591:0:61515:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051821 expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.149614:0:61515:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051821 expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.149622:0:61515:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051821 expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.149629:0:61515:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051821 expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764725.149641:0:61515:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea7b80 x1862022098579968/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/400 e 0 to 0 dl 1775764741 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764725.149651:0:61515:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051821 expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764725.149660:0:61515:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051821 expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.149670:0:61515:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/1 (0 0) 00000080:00200000:2.0:1775764725.149675:0:61515:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x1:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764725.149688:0:61515:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) for lock 0x73cfd86cfc17bc11 00000080:00010000:2.0:1775764725.149695:0:61515:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc11 from it ffffa60b01eebc10 00010000:00010000:2.0:1775764725.149696:0:61515:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051821 expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.149702:0:61515:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051821 expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.149718:0:61515:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051821 expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.149725:0:61515:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051821 expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764725.149730:0:61515:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1:0x0](00000000492670d3) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051821 expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764725.149781:0:61515:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1:0x0]: layout version change: 4294967294 -> 5 00010000:00010000:2.0:1775764725.149797:0:61515:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051821 expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.149801:0:61515:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051821 expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:2.0:1775764725.150185:0:61515:0:(file.c:2454:do_file_read_iter()) START file f22.sanity-pcc:[0x200003ab4:0x1:0x0], ppos: 0, count: 1048576 00000080:00200000:2.0:1775764725.150216:0:61515:0:(rw.c:1928:ll_readpage()) fast read pgno: 0 00000080:00200000:2.0:1775764725.150239:0:61515:0:(file.c:2058:ll_file_io_generic()) f22.sanity-pcc: read ppos: 0, bytes: 1048576 00000080:00200000:2.0:1775764725.150244:0:61515:0:(file.c:1815:ll_io_set_mirror()) f22.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764725.150249:0:61515:0:(cl_io.c:214:cl_io_rw_init()) header@0000000083af2673[0x110, 1, [0x200003ab4:0x1:0x0] hash] 00000020:00200000:2.0:1775764725.150251:0:61515:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 1048576) 0 0 00000080:00200000:2.0:1775764725.150257:0:61515:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764725.150275:0:61515:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:2.0:1775764725.150279:0:61515:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764725.150312:0:61515:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:2.0:1775764725.150332:0:61515:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764725.150340:0:61515:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24db0, osc lock ffff8b238a3f90a0, flags 0x0 00010000:00010000:2.0:1775764725.150418:0:61515:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.150431:0:61515:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.150443:0:61515:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764725.150512:0:61515:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61515:1862022098580096:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764725.150587:0:61515:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764725.153200:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098579456:192.168.202.128@tcp:57:lhsmtool_posix.0 00000080:00200008:2.0:1775764725.153305:0:61516:0:(file.c:447:ll_file_release()) START file . :VOLATILE:0000:6F3D6CF2:fd=09:[0x200003ab4:0x5:0x0](ffff8b2390ec42c8), flags 500002 00000002:00100000:2.0:1775764725.153368:0:61516:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea7100 x1862022098575744/t34359739087(34359739087) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 608/664 e 0 to 0 dl 1775764740 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764725.153393:0:61516:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764725.153398:0:61516:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4a80 x1862022098580352/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764725.153418:0:61516:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098580352:192.168.202.128@tcp:35:lhsmtool_posix.0 00000100:00100000:2.0:1775764725.154231:0:61516:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764725.155299:0:61515:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61515:1862022098580096:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764725.155327:0:61515:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 4/1,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.155351:0:61515:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.155365:0:61515:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.155394:0:61515:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1775764725.155409:0:61515:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 13, setting rss=13; kms=13, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 4/2,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.155421:0:61515:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.155429:0:61515:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764725.155499:0:61515:0:(vvp_io.c:826:vvp_io_read_start()) f22.sanity-pcc: read [0, 1048576) 00000001:00010000:2.0:1775764725.155525:0:61515:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x1:0x0] 00000008:00010000:2.0:1775764725.155536:0:61515:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764725.155540:0:61515:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24430, osc lock ffff8b238a3f8390, flags 0x400001002 00010000:00010000:2.0:1775764725.155556:0:61515:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 4/2,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.155574:0:61515:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 4/2,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1775764725.155583:0:61515:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764725.155587:0:61515:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 7/2,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.155593:0:61515:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 6/1,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1775764725.155608:0:61515:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764725.155616:0:61515:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1:0x0] updating i_size 13 i_blocks 11 00000020:00010000:2.0:1775764725.155620:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c6d8 (ffff8b23812ec758/0) at cl_lock_release():245 00000020:00010000:2.0:1775764725.155623:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c6d8 (ffff8b23812ec758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764725.155627:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24430 (ffff8b23812ec1b8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764725.155631:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c6d8 (ffff8b23812ec758/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764725.155633:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24430 (ffff8b23812ec758/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764725.155646:0:61515:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:0 00000008:00010000:2.0:1775764725.155702:0:61515:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b2383c9e3c0 00000008:00010000:2.0:1775764725.155767:0:46654:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b2383c9e3c0 00010000:00010000:2.0:1775764725.155789:0:46654:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00100000:2.0:1775764725.155819:0:46654:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea7b80 - object 0x280000400:280 offset 0<>4096 00000100:00100000:2.0:1775764725.161923:0:61516:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61516:1862022098580352:192.168.202.128@tcp:35:lhsmtool_posix.0 00000002:00100000:2.0:1775764725.161955:0:61516:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea7100 x1862022098575744/t34359739087(34359739087) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 608/664 e 0 to 0 dl 1775764740 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000020:00010000:2.0:1775764725.162541:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b5d8 (ffff8b23812ec758/0) at cl_lock_release():245 00000020:00010000:2.0:1775764725.162545:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b5d8 (ffff8b23812ec758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764725.162548:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24db0 (ffff8b23812ec1b8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764725.162556:0:61515:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.162568:0:61515:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1775764725.162584:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b5d8 (ffff8b23812ec758/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764725.162587:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24db0 (ffff8b23812ec758/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764725.162604:0:61515:0:(vvp_io.c:517:vvp_io_advance()) advancing 13 bytes 00000080:00200000:2.0:1775764725.162635:0:61515:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1:0x0] ignore/verify layout 0/0, layout version 5 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764725.162639:0:61515:0:(file.c:2206:ll_file_io_generic()) f22.sanity-pcc: 1 io complete with rc: 0, result: 13, restart: 0 00000080:00200000:2.0:1775764725.162643:0:61515:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 13 00000080:00200008:2.0:1775764725.162697:0:61515:0:(file.c:2454:do_file_read_iter()) START file f22.sanity-pcc:[0x200003ab4:0x1:0x0], ppos: 13, count: 1048576 00000080:00200000:2.0:1775764725.162714:0:61515:0:(file.c:2058:ll_file_io_generic()) f22.sanity-pcc: read ppos: 13, bytes: 1048576 00000080:00200000:2.0:1775764725.162717:0:61515:0:(file.c:1815:ll_io_set_mirror()) f22.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764725.162722:0:61515:0:(cl_io.c:214:cl_io_rw_init()) header@0000000083af2673[0x110, 2, [0x200003ab4:0x1:0x0] hash] 00000020:00200000:2.0:1775764725.162723:0:61515:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [13, 1048589) 0 0 00000080:00200000:2.0:1775764725.162727:0:61515:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764725.162740:0:61515:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 13 chunk: [13, 1048589) 1048589, 1048576 00020000:00200000:2.0:1775764725.162743:0:61515:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764725.162762:0:61515:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 256] 00000008:00010000:2.0:1775764725.162771:0:61515:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764725.162774:0:61515:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24db0, osc lock ffff8b238a3f90a0, flags 0x0 00010000:00010000:2.0:1775764725.162790:0:61515:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.162800:0:61515:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1052671) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.162808:0:61515:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.162812:0:61515:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.162816:0:61515:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764725.162822:0:61515:0:(vvp_io.c:826:vvp_io_read_start()) f22.sanity-pcc: read [13, 1048589) 00000001:00010000:2.0:1775764725.162832:0:61515:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x1:0x0] 00000008:00010000:2.0:1775764725.162835:0:61515:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764725.162837:0:61515:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24430, osc lock ffff8b238a3f8390, flags 0x400001002 00010000:00010000:2.0:1775764725.162840:0:61515:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 4/2,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.162846:0:61515:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 4/2,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1775764725.162850:0:61515:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764725.162851:0:61515:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 7/2,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.162860:0:61515:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 6/1,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1775764725.162863:0:61515:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764725.162867:0:61515:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1:0x0] updating i_size 13 i_blocks 11 00000020:00010000:2.0:1775764725.162868:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c6d8 (ffff8b23812ec758/0) at cl_lock_release():245 00000020:00010000:2.0:1775764725.162870:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c6d8 (ffff8b23812ec758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764725.162871:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24430 (ffff8b23812ec1b8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764725.162872:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c6d8 (ffff8b23812ec758/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764725.162873:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24430 (ffff8b23812ec758/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764725.162877:0:61515:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:13 00000020:00010000:2.0:1775764725.162880:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b5d8 (ffff8b23812ec758/0) at cl_lock_release():245 00000020:00010000:2.0:1775764725.162881:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b5d8 (ffff8b23812ec758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764725.162882:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24db0 (ffff8b23812ec1b8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764725.162885:0:61515:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.162889:0:61515:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026ad166b/0x73cfd86cfc17bc18 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051828 expref: -99 pid: 61515 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00010000:2.0:1775764725.162894:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b5d8 (ffff8b23812ec758/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764725.162895:0:61515:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24db0 (ffff8b23812ec758/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764725.162897:0:61515:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1775764725.162906:0:61515:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1:0x0] ignore/verify layout 0/0, layout version 5 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764725.162907:0:61515:0:(file.c:2206:ll_file_io_generic()) f22.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1775764725.162909:0:61515:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:2.0:1775764725.163054:0:61515:0:(file.c:447:ll_file_release()) START file f22.sanity-pcc:[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), flags 100000 00000002:00100000:2.0:1775764725.163094:0:61515:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b2382dac000 x1862022098573568/t34359739084(34359739084) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764740 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764725.163101:0:61515:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764725.163104:0:61515:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7100 x1862022098580736/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764725.163120:0:61515:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61515:1862022098580736:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764725.163187:0:61515:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764725.169782:0:61515:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61515:1862022098580736:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764725.169812:0:61515:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b2382dac000 x1862022098573568/t34359739084(34359739084) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764740 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764725.180909:0:61517:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764725.181006:0:61517:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764725.181068:0:61517:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 61517 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.181078:0:61517:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61517 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.181086:0:61517:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61517 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764725.181107:0:61517:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61517:1862022098580864:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764725.181231:0:61517:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384904840 going to sleep for 11 seconds 00000100:00100000:2.0:1775764725.184139:0:61517:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61517:1862022098580864:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764725.184250:0:61517:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.184264:0:61517:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.184277:0:61517:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.184290:0:61517:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.184305:0:61517:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.184325:0:61517:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.184334:0:61517:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764725.184346:0:61517:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864fd500 x1862022098580864/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764741 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764725.184361:0:61517:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.184388:0:61517:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764725.184397:0:61517:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764725.184429:0:61517:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764725.184435:0:61517:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bc1f 00000080:00010000:2.0:1775764725.184442:0:61517:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764725.184445:0:61517:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc1f from it ffffa60b01f13bf8 00010000:00010000:2.0:1775764725.184448:0:61517:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.184668:0:61517:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764725.184701:0:61517:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764725.184725:0:61517:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f22.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b2386292900, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1775764725.184734:0:61517:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f22.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open 00000002:00010000:2.0:1775764725.184762:0:61517:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f22.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:2.0:1775764725.184816:0:61517:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc26 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 61517 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.184823:0:61517:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc26 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61517 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764725.184834:0:61517:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764725.184841:0:61517:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7100 x1862022098581120/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764725.184849:0:61517:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc26 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61517 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764725.184870:0:61517:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61517:1862022098581120:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764725.185064:0:61517:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764725.187748:0:61517:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61517:1862022098581120:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764725.187776:0:61517:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc26 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61517 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.187792:0:61517:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc26 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 61517 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.187799:0:61517:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc26 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 61517 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.187805:0:61517:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc26 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 61517 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.187813:0:61517:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc26 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 61517 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.187820:0:61517:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6ebc00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764725.187825:0:61517:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc26 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 61517 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.187830:0:61517:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc26 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 61517 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.187837:0:61517:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc26 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 61517 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.187844:0:61517:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764725.187850:0:61517:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc26 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 61517 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.187856:0:61517:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc26 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 61517 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764725.187875:0:61517:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8b23b9ea7100 x1862022098581120/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/608 e 0 to 0 dl 1775764741 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764725.187890:0:61517:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764725.188062:0:61517:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764725.188090:0:61517:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764725.188108:0:61517:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.188123:0:61517:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764725.188130:0:61517:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764725.188137:0:61517:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764725.188140:0:61517:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bc1f 00000080:00010000:2.0:1775764725.188146:0:61517:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc1f from it ffffa60b01f13bb0 00010000:00010000:2.0:1775764725.188149:0:61517:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.188157:0:61517:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764725.188174:0:61517:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764725.188185:0:61517:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764725.188220:0:61517:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086915 arg=19463d0 00000100:00100000:2.0:1775764725.188283:0:61517:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61517:1862022098581248:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764725.188412:0:61517:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764725.198513:0:61517:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61517:1862022098581248:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764725.198726:0:61517:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:2.0:1775764725.291527:0:61527:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764725.291555:0:61527:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764725.291585:0:61527:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.291612:0:61527:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764725.291620:0:61527:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764725.291629:0:61527:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764725.291634:0:61527:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bc1f 00000080:00010000:2.0:1775764725.291641:0:61527:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc1f from it ffffa60b01f67bb0 00010000:00010000:2.0:1775764725.291645:0:61527:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.291655:0:61527:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764725.291671:0:61527:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764725.291679:0:61527:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764725.291703:0:61527:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7fffd3efb260 00000100:00100000:2.0:1775764725.291767:0:61527:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61527:1862022098581376:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764725.291849:0:61527:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386d80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764725.299796:0:61527:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61527:1862022098581376:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764725.299901:0:61527:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:2.0:1775764725.381167:0:61534:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764725.381207:0:61534:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764725.381229:0:61534:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764725.381247:0:61534:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764725.381256:0:61534:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764725.381266:0:61534:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764725.381270:0:61534:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bc1f 00000080:00010000:2.0:1775764725.381278:0:61534:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc1f from it ffffa60b01f47bf8 00010000:00010000:2.0:1775764725.381282:0:61534:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764725.381895:0:61534:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764725.381920:0:61534:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764725.381942:0:61534:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f22.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b2384ca2f00, open_flags 8000, mode 0 opened 0 00000080:00200000:2.0:1775764725.381951:0:61534:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f22.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open 00000002:00010000:2.0:1775764725.381973:0:61534:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f22.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100001 00010000:00010000:2.0:1775764725.382039:0:61534:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 61534 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.382048:0:61534:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61534 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764725.382061:0:61534:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764725.382076:0:61534:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7100 x1862022098581632/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764725.382086:0:61534:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61534 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764725.382111:0:61534:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:61534:1862022098581632:192.168.202.128@tcp:101:cat.0 00000100:00100000:2.0:1775764725.382281:0:61534:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386d80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764725.393792:0:61534:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:61534:1862022098581632:192.168.202.128@tcp:101:cat.0 00010000:00010000:2.0:1775764725.393822:0:61534:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051898 expref: -99 pid: 61534 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.393839:0:61534:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051898 expref: -99 pid: 61534 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.393847:0:61534:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051898 expref: -99 pid: 61534 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.393858:0:61534:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051898 expref: -99 pid: 61534 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.393869:0:61534:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051898 expref: -99 pid: 61534 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.393881:0:61534:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051898 expref: -99 pid: 61534 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.393889:0:61534:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051898 expref: -99 pid: 61534 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.393896:0:61534:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051898 expref: -99 pid: 61534 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764725.393906:0:61534:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23b9ea7100 x1862022098581632/t34359739099(34359739099) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764741 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764725.393918:0:61534:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7100 x1862022098581632/t34359739099(34359739099) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764741 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764725.393930:0:61534:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051898 expref: -99 pid: 61534 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764725.393942:0:61534:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051898 expref: -99 pid: 61534 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.393952:0:61534:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/1 (0 0) 00000080:00200000:2.0:1775764725.393960:0:61534:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764725.394050:0:61534:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0x1:0x0](ffff8b2390ec21c8): rc = 0 00000080:00200000:2.0:1775764725.394072:0:61534:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1:0x0]: layout version change: 5 -> 5 00000080:00010000:2.0:1775764725.394083:0:61534:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) for lock 0x73cfd86cfc17bc2d 00000080:00010000:2.0:1775764725.394094:0:61534:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) 00000080:00200008:2.0:1775764725.394109:0:61534:0:(file.c:1057:ll_file_open()) START file f22.sanity-pcc:[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), flags 100000 00000002:00100000:2.0:1775764725.394130:0:61534:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7100 x1862022098581632/t34359739099(34359739099) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764741 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764725.394176:0:61534:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc2d from it ffff8b238504c360 00010000:00010000:2.0:1775764725.394180:0:61534:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051898 expref: -99 pid: 61534 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.394189:0:61534:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051898 expref: -99 pid: 61534 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764725.394226:0:61534:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), inode mode 81a4 mask 44 00000080:00200008:2.0:1775764725.394253:0:61534:0:(file.c:6226:ll_getattr_dentry()) START file f22.sanity-pcc:[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764725.394259:0:61534:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x1:0x0](ffff8b2390ec21c8),name=f22.sanity-pcc 00000002:00010000:2.0:1775764725.394281:0:61534:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764725.394290:0:61534:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051898 expref: -99 pid: 61534 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.394302:0:61534:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051898 expref: -99 pid: 61534 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764725.394315:0:61534:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764725.394321:0:61534:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) 00000080:00010000:2.0:1775764725.394324:0:61534:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2390ec21c8) for lock 0x73cfd86cfc17bc2d 00000080:00010000:2.0:1775764725.394328:0:61534:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc2d from it ffffa60b01f47cd8 00010000:00010000:2.0:1775764725.394331:0:61534:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051898 expref: -99 pid: 61534 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.394339:0:61534:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bc2d lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051898 expref: -99 pid: 61534 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:2.0:1775764725.394503:0:61534:0:(file.c:2454:do_file_read_iter()) START file f22.sanity-pcc:[0x200003ab4:0x1:0x0], ppos: 0, count: 4194304 00000080:00200008:2.0:1775764725.394585:0:61534:0:(file.c:2454:do_file_read_iter()) START file f22.sanity-pcc:[0x200003ab4:0x1:0x0], ppos: 13, count: 4194304 00000080:00200008:2.0:1775764725.394677:0:61534:0:(file.c:447:ll_file_release()) START file f22.sanity-pcc:[0x200003ab4:0x1:0x0](ffff8b2390ec21c8), flags 100000 00000002:00100000:2.0:1775764725.394725:0:61534:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea7100 x1862022098581632/t34359739099(34359739099) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764741 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cat.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764725.394736:0:61534:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764725.394741:0:61534:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6680 x1862022098581888/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764725.394767:0:61534:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:61534:1862022098581888:192.168.202.128@tcp:35:cat.0 00000100:00100000:2.0:1775764725.394846:0:61534:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386d80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764725.398816:0:61534:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:61534:1862022098581888:192.168.202.128@tcp:35:cat.0 00000002:00100000:2.0:1775764725.398854:0:61534:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea7100 x1862022098581632/t34359739099(34359739099) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764741 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cat.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764725.489135:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200000:2.0:1775764725.533446:0:61541:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764725.534152:0:61541:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764725.534173:0:61541:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764725.534193:0:61541:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200000:2.0:1775764725.534201:0:61541:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764725.534212:0:61541:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764725.534216:0:61541:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bc1f 00000080:00010000:2.0:1775764725.534224:0:61541:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc1f from it ffffa60b01f4fbf8 00010000:00010000:2.0:1775764725.534228:0:61541:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764725.534236:0:61541:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:2.0:1775764725.534254:0:61541:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764725.534271:0:61541:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f22.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b23a00a7200, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1775764725.534278:0:61541:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f22.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open 00000002:00010000:2.0:1775764725.534292:0:61541:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f22.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:2.0:1775764725.534344:0:61541:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc3b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 61541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.534352:0:61541:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc3b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61541 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764725.534364:0:61541:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764725.534377:0:61541:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6680 x1862022098582528/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764725.534388:0:61541:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc3b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61541 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764725.534414:0:61541:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61541:1862022098582528:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764725.535197:0:61541:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386d80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764725.544025:0:61541:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61541:1862022098582528:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764725.544047:0:61541:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc3b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.544060:0:61541:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc3b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 61541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.544066:0:61541:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc3b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 61541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.544072:0:61541:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc3b lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 61541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.544080:0:61541:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc3b lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 61541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.544086:0:61541:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6eaa00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764725.544090:0:61541:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc3b lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 61541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.544097:0:61541:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc3b lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 61541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.544104:0:61541:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc3b lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 61541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.544112:0:61541:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764725.544118:0:61541:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc3b lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 61541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.544123:0:61541:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc3b lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 61541 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764725.544141:0:61541:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8b23b9ea6680 x1862022098582528/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/608 e 0 to 0 dl 1775764741 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764725.544155:0:61541:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764725.544281:0:61541:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764725.544300:0:61541:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764725.544313:0:61541:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1775764725.544325:0:61541:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:2.0:1775764725.544331:0:61541:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764725.544336:0:61541:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764725.544340:0:61541:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bc1f 00000080:00010000:2.0:1775764725.544345:0:61541:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc1f from it ffffa60b01f4fbb0 00010000:00010000:2.0:1775764725.544348:0:61541:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1775764725.544356:0:61541:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17bc1f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305183d expref: -99 pid: 61517 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:2.0:1775764725.544388:0:61541:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764725.544399:0:61541:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764725.544429:0:61541:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086915 arg=152d3d0 00000100:00100000:2.0:1775764725.544524:0:61541:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61541:1862022098582656:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764725.544604:0:61541:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386d80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764725.550152:0:61541:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61541:1862022098582656:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764725.550375:0:61541:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:2.0:1775764725.566324:0:61542:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:2.0:1775764725.566353:0:61542:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764725.566419:0:61542:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 61542 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.566431:0:61542:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61542 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764725.566442:0:61542:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61542 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764725.566505:0:61542:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job rm:82781e18-e0e3-4f9f-9a2b-041bf0344414:61542:1862022098582784:192.168.202.128@tcp:101:rm.0 00000100:00100000:2.0:1775764725.566629:0:61542:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386d80 going to sleep for 11 seconds 00000100:00100000:1.0:1775764725.570895:0:61542:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job rm:82781e18-e0e3-4f9f-9a2b-041bf0344414:61542:1862022098582784:192.168.202.128@tcp:101:rm.0 00010000:00010000:1.0:1775764725.570918:0:61542:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.570932:0:61542:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.570956:0:61542:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.570967:0:61542:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.570980:0:61542:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.570987:0:61542:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.570994:0:61542:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764725.571005:0:61542:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea6680 x1862022098582784/t0(0) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764741 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764725.571017:0:61542:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.571030:0:61542:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2387c07800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1775764725.571037:0:61542:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764725.571065:0:61542:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:1.0:1775764725.571070:0:61542:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17bc42 00000080:00010000:1.0:1775764725.571075:0:61542:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:1.0:1775764725.571078:0:61542:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc42 from it ffffa60b01f57b58 00010000:00010000:1.0:1775764725.571081:0:61542:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.571095:0:61542:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764725.571120:0:61542:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148), inode mode 41ed mask 1 00000080:00200000:1.0:1775764725.571141:0:61542:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=f22.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2389909148), flags=8 00000080:00200000:1.0:1775764725.571145:0:61542:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f22.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2389909148), intent=getattr 00000002:00010000:1.0:1775764725.571638:0:61542:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f22.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764725.571686:0:61542:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 61542 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.571694:0:61542:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61542 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.571702:0:61542:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61542 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764725.571728:0:61542:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job rm:82781e18-e0e3-4f9f-9a2b-041bf0344414:61542:1862022098582912:192.168.202.128@tcp:101:rm.0 00000100:00100000:1.0:1775764725.571863:0:61542:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384904b40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764725.579049:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119045760 00000100:00100000:1.0:1775764725.579058:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119045760 00000100:00100000:1.0:1775764725.579141:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 227 00000100:00100000:1.0:1775764725.579225:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dac380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119045760:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764725.579244:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17bc34 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x1:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e030518ad expref: -99 pid: 61540 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764725.579321:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764725.579327:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dac380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119045760:12345-192.168.202.128@tcp:104: Request processed in 179us (459us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764725.579336:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 227 00010000:00010000:2.0:1775764725.579388:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764725.579394:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17bc34 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x1:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e030518ad expref: -99 pid: 61540 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.579404:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5ec00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764725.579409:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17bc34 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x1:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e030518ad expref: -99 pid: 61540 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.579418:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17bc34 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x1:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e030518ad expref: -99 pid: 61540 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.579426:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17bc34 lrc: 3/0,0 mode: CW/CW res: [0x200003ab4:0x1:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x57b8ff0e030518ad expref: -99 pid: 61540 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000002:00100000:2.0:1775764725.579520:0:46150:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864f0380 x1862022098582144/t34359739101(34359739101) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764741 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764725.579532:0:46150:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764725.579537:0:46150:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7100 x1862022098583168/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764725.579561:0:46150:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_02:fcf15866-4739-4595-8fe1-44abde7ee9d7:46150:1862022098583168:192.168.202.128@tcp:35:ldlm_bl.0 00000100:00100000:2.0:1775764725.579623:0:46150:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764725.584122:0:46150:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_02:fcf15866-4739-4595-8fe1-44abde7ee9d7:46150:1862022098583168:192.168.202.128@tcp:35:ldlm_bl.0 00000002:00100000:2.0:1775764725.584234:0:46150:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864f0380 x1862022098582144/t34359739101(34359739101) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764741 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764725.584311:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17bc34 lrc: 2/0,0 mode: --/CW res: [0x200003ab4:0x1:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030518ad expref: -99 pid: 61540 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.584320:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764725.587232:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17bc34 lrc: 1/0,0 mode: --/CW res: [0x200003ab4:0x1:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030518ad expref: -99 pid: 61540 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.587249:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17bc34 lrc: 0/0,0 mode: --/CW res: [0x200003ab4:0x1:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030518ad expref: -99 pid: 61540 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.587271:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764725.587273:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764725.587276:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764725.590168:0:61542:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job rm:82781e18-e0e3-4f9f-9a2b-041bf0344414:61542:1862022098582912:192.168.202.128@tcp:101:rm.0 00010000:00010000:1.0:1775764725.590195:0:61542:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.590216:0:61542:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.590224:0:61542:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.590235:0:61542:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.590245:0:61542:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.590256:0:61542:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.590264:0:61542:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.590270:0:61542:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764725.590280:0:61542:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864fc380 x1862022098582912/t0(0) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 584/640 e 0 to 0 dl 1775764741 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764725.590291:0:61542:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764725.590303:0:61542:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.590312:0:61542:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2387c07800 type 13 mode 4 res 8589949620/1 (0 0) 00000080:00200000:1.0:1775764725.590318:0:61542:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764725.590355:0:61542:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab4:0x1:0x0](ffff8b2389a65b88) 00000080:00200000:1.0:1775764725.590513:0:61542:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1:0x0]: layout version change: 4294967294 -> 5 00000080:00010000:1.0:1775764725.590525:0:61542:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2389a65b88) for lock 0x73cfd86cfc17bc49 00000080:00010000:1.0:1775764725.590541:0:61542:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2389a65b88) 00000080:00010000:1.0:1775764725.590545:0:61542:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc49 from it ffffa60b01f57b20 00010000:00010000:1.0:1775764725.590548:0:61542:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.590557:0:61542:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:1.0:1775764725.590603:0:61542:0:(file.c:6226:ll_getattr_dentry()) START file f22.sanity-pcc:[0x200003ab4:0x1:0x0](ffff8b2389a65b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764725.590610:0:61542:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x1:0x0](ffff8b2389a65b88),name=f22.sanity-pcc 00000002:00010000:1.0:1775764725.590630:0:61542:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764725.590639:0:61542:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764725.590649:0:61542:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764725.590662:0:61542:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764725.590667:0:61542:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2389a65b88) 00000080:00010000:1.0:1775764725.590670:0:61542:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1:0x0](ffff8b2389a65b88) for lock 0x73cfd86cfc17bc49 00000080:00010000:1.0:1775764725.590673:0:61542:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc49 from it ffffa60b01f57cb8 00010000:00010000:1.0:1775764725.590676:0:61542:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764725.590684:0:61542:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764725.590709:0:61542:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764725.590719:0:61542:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x1:0x0] 00000008:00010000:1.0:1775764725.590747:0:61542:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764725.590751:0:61542:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5eb0, osc lock ffff8b2382cf6850, flags 0x400001002 00000100:00080000:1.0:1775764725.590771:0:61542:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 1840 00000100:00080000:1.0:1775764725.590781:0:61542:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b238a3bf000: import ffff8b23a00b5000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775764725.590793:0:61542:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 3407872, lost: 0. 00000100:00100000:1.0:1775764725.590817:0:61542:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098583424/t0(0) o8->lustre-OST0001-osc-ffff8b238a3bf000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775764725.590866:0:61542:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b238a3bf000 lock: 00000000da82aaac/0x73cfd86cfc17bc50 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 61542 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.590876:0:61542:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8b238a3bf000 lock: 00000000da82aaac/0x73cfd86cfc17bc50 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 61542 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1775764725.590909:0:61542:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:2.0:1775764725.591042:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098583424:192.168.202.128@tcp:8:rm.0 00000100:00100000:2.0:1775764725.594848:0:2375:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b238a3bf000: committing for last_committed 0 gen 2 00000100:00080000:2.0:1775764725.594862:0:2375:0:(import.c:1130:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffff8b238a3bf000: connect to target with instance 8 00000100:00080000:2.0:1775764725.594871:0:2375:0:(import.c:958:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffff8b238a3bf000: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:2.0:1775764725.594881:0:2375:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1775764725.594887:0:2375:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1775764725.594892:0:2375:0:(import.c:1211:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:2.0:1775764725.594896:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b5000 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00000080:00000004:2.0:1775764725.594903:0:2375:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1775764725.594905:0:2375:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1775764725.594916:0:2375:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 000000001f5dc83a): req@ffff8b23864fc380 x1862022098583552/t0(0) o101->lustre-OST0001-osc-ffff8b238a3bf000@192.168.202.128@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764725.594945:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098583424:192.168.202.128@tcp:8:rm.0 00010000:00010000:1.0:1775764725.598089:0:2377:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b238a3bf000 lock: 00000000da82aaac/0x73cfd86cfc17bc50 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 61542 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.598116:0:2377:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8b238a3bf000 lock: 00000000da82aaac/0x73cfd86cfc17bc50 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051916 expref: -99 pid: 61542 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.598130:0:2377:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b238a3bf000 lock: 00000000da82aaac/0x73cfd86cfc17bc50 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051916 expref: -99 pid: 61542 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.598146:0:2377:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8b238a3bf000 lock: 00000000da82aaac/0x73cfd86cfc17bc50 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051916 expref: -99 pid: 61542 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1775764725.598175:0:2377:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 13, setting rss=13; kms=13, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b238a3bf000 lock: 00000000da82aaac/0x73cfd86cfc17bc50 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051916 expref: -99 pid: 61542 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1775764725.598187:0:2377:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1775764725.598250:0:2377:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b238a3bf000 lock: 00000000da82aaac/0x73cfd86cfc17bc50 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051916 expref: -99 pid: 61542 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.598258:0:2377:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b238a3bf000 lock: 00000000da82aaac/0x73cfd86cfc17bc50 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051916 expref: -99 pid: 61542 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.598265:0:2377:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b238a3bf000 lock: 00000000da82aaac/0x73cfd86cfc17bc50 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051916 expref: -99 pid: 61542 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.598275:0:2377:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b238a3bf000 lock: 00000000da82aaac/0x73cfd86cfc17bc50 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051916 expref: -99 pid: 61542 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1775764725.598295:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:0:1862022098583552:192.168.202.128@tcp:101:rm.0 00000080:00200000:2.0:1775764725.599005:0:61542:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1:0x0] updating i_size 13 i_blocks 11 00000020:00010000:2.0:1775764725.599014:0:61542:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1b478 (ffff8b2387cba248/0) at cl_lock_release():245 00000020:00010000:2.0:1775764725.599017:0:61542:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1b478 (ffff8b2387cba248/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764725.599023:0:61542:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5eb0 (ffff8b2387cba6c8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764725.599028:0:61542:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1b478 (ffff8b2387cba248/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764725.599031:0:61542:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5eb0 (ffff8b2387cba248/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764725.599067:0:61542:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1:0x0] ignore/verify layout 0/1, layout version 5 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764725.599138:0:61542:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:2.0:1775764725.599235:0:61542:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764725.599254:0:61542:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.599269:0:61542:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764725.599277:0:61542:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764725.599285:0:61542:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:2.0:1775764725.599288:0:61542:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17bc42 00000080:00010000:2.0:1775764725.599295:0:61542:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc42 from it ffffa60b01f57c10 00010000:00010000:2.0:1775764725.599298:0:61542:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764725.599306:0:61542:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764725.599318:0:61542:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148), inode mode 41ed mask 1 00000080:00200000:2.0:1775764725.599327:0:61542:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f22.sanity-pcc, flags=0 00000080:00200000:2.0:1775764725.599334:0:61542:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:2.0:1775764725.599341:0:61542:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764725.599346:0:61542:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.599354:0:61542:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764725.599360:0:61542:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764725.599365:0:61542:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:2.0:1775764725.599368:0:61542:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17bc42 00000080:00010000:2.0:1775764725.599554:0:61542:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc42 from it ffffa60b01f57d68 00010000:00010000:2.0:1775764725.599557:0:61542:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.599565:0:61542:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764725.599575:0:61542:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148), inode mode 41ed mask 3 00000080:00200000:2.0:1775764725.599581:0:61542:0:(namei.c:2405:ll_unlink()) VFS Op:name=f22.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2389909148) 00010000:00010000:2.0:1775764725.599657:0:61542:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764725.599685:0:61542:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764725.599704:0:61542:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1:0x0]: losing layout lock 00010000:00010000:2.0:1775764725.599744:0:61542:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.599752:0:61542:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764725.599758:0:61542:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1775764725.599761:0:61542:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bc49 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051908 expref: -99 pid: 61542 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764725.599779:0:61542:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc42 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030518ec expref: -99 pid: 61542 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:2.0:1775764725.599806:0:61542:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b238a3bf000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764725.599817:0:61542:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6a00 x1862022098583808/t0(0) o36->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 496/1032 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764725.599843:0:61542:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job rm:82781e18-e0e3-4f9f-9a2b-041bf0344414:61542:1862022098583808:192.168.202.128@tcp:36:rm.0 00000100:00100000:2.0:1775764725.600088:0:61542:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764725.603645:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:2.0:1775764725.619857:0:61542:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job rm:82781e18-e0e3-4f9f-9a2b-041bf0344414:61542:1862022098583808:192.168.202.128@tcp:36:rm.0 00000080:00200000:2.0:1775764725.620108:0:61542:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764725.620122:0:61542:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764725.620182:0:61542:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764725.620197:0:61542:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200003ab4:0x1:0x0](ffff8b2389a65b88) 00000008:00010000:2.0:1775764725.620237:0:61542:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 13 blocks 11 [cma]time: 1775764723 1775764721 1775764724 by oinfo size 13 blocks 11 [cma]time 1775764723 1775764721 1775764724 ns: lustre-OST0001-osc-ffff8b238a3bf000 lock: 00000000da82aaac/0x73cfd86cfc17bc50 lrc: 1/0,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051916 expref: -99 pid: 61542 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1775764725.667767:0:61548:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:61548:1862022098584064:192.168.202.128@tcp:101:bash.0 00010000:00010000:1.0:1775764725.667796:0:61548:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bc57 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051940 expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.667810:0:61548:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bc57 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051940 expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.667823:0:61548:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bc57 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051940 expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.667834:0:61548:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bc57 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051940 expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.667847:0:61548:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bc57 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051940 expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.667856:0:61548:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bc57 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051940 expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.667863:0:61548:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bc57 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051940 expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764725.667875:0:61548:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b2382dac700 x1862022098584064/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764741 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764725.667890:0:61548:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bc57 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051940 expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.667906:0:61548:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1775764725.667914:0:61548:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764725.667949:0:61548:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:1.0:1775764725.667956:0:61548:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bc57 00000080:00010000:1.0:1775764725.667962:0:61548:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764725.667966:0:61548:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc57 from it ffffa60b01f77bf8 00010000:00010000:1.0:1775764725.667969:0:61548:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bc57 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051940 expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.667979:0:61548:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bc57 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051940 expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764725.668006:0:61548:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764725.668032:0:61548:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764725.668045:0:61548:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764725.668054:0:61548:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bc57 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051940 expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764725.668065:0:61548:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bc57 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051940 expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764725.668078:0:61548:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764725.668083:0:61548:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764725.668085:0:61548:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bc57 00000080:00010000:1.0:1775764725.668089:0:61548:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc57 from it ffffa60b01f77b38 00010000:00010000:1.0:1775764725.668092:0:61548:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bc57 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051940 expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764725.668099:0:61548:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bc57 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051940 expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764725.668106:0:61548:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:1.0:1775764725.668111:0:61548:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f22.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b2383eaf500, open_flags 8241, mode 81b6 opened 0 00000080:00200000:1.0:1775764725.668119:0:61548:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f22.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000002:00010000:1.0:1775764725.668139:0:61548:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f22.sanity-pcc,[0x200003ab4:0x6:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0101102 00010000:00010000:1.0:1775764725.668150:0:61548:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bc57 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03051940 expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764725.668200:0:61548:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bc57 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051940 expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764725.668208:0:61548:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764725.668210:0:61548:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bc57 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051940 expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764725.668245:0:61548:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 61548 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.668255:0:61548:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61548 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764725.668267:0:61548:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764725.668273:0:61548:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fdc00 x1862022098584320/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764725.668281:0:61548:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61548 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764725.668304:0:61548:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:61548:1862022098584320:192.168.202.128@tcp:101:bash.0 00000100:00100000:1.0:1775764725.668447:0:61548:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384904a80 going to sleep for 11 seconds 00000100:00100000:1.0:1775764725.675826:0:61548:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:61548:1862022098584320:192.168.202.128@tcp:101:bash.0 00010000:00010000:1.0:1775764725.675856:0:61548:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.675893:0:61548:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.675902:0:61548:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.675913:0:61548:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.675925:0:61548:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.675944:0:61548:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.675950:0:61548:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.675957:0:61548:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764725.675969:0:61548:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8b23864fdc00 x1862022098584320/t34359739104(34359739104) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764741 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764725.675981:0:61548:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fdc00 x1862022098584320/t34359739104(34359739104) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764741 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764725.675996:0:61548:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764725.676009:0:61548:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.676022:0:61548:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/6 (0 0) 00000080:00200000:1.0:1775764725.676030:0:61548:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764725.676050:0:61548:0:(namei.c:1620:ll_create_it()) VFS Op:name=f22.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000080:00200000:1.0:1775764725.676079:0:61548:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148) 00000080:00200000:1.0:1775764725.676217:0:61548:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x6:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1775764725.676226:0:61548:0:(namei.c:1586:ll_create_node()) setting l_ast_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) 00000080:00010000:1.0:1775764725.676230:0:61548:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) for lock 0x73cfd86cfc17bc5e 00000080:00010000:1.0:1775764725.676239:0:61548:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) 00000080:00200008:1.0:1775764725.676255:0:61548:0:(file.c:1057:ll_file_open()) START file f22.sanity-pcc:[0x200003ab4:0x6:0x0](ffff8b2389a61148), flags 101101 00000002:00100000:1.0:1775764725.676273:0:61548:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fdc00 x1862022098584320/t34359739104(34359739104) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764741 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1775764725.676315:0:61548:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc5e from it ffff8b2389ed6960 00010000:00010000:1.0:1775764725.676320:0:61548:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.676328:0:61548:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764725.676368:0:61548:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148), inode mode 81a4 mask 40 00000080:00200008:1.0:1775764725.676500:0:61548:0:(file.c:6226:ll_getattr_dentry()) START file f22.sanity-pcc:[0x200003ab4:0x6:0x0](ffff8b2389a61148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764725.676507:0:61548:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148),name=f22.sanity-pcc 00000002:00010000:1.0:1775764725.676531:0:61548:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764725.676542:0:61548:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764725.676553:0:61548:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764725.676560:0:61548:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764725.676565:0:61548:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) 00000080:00010000:1.0:1775764725.676569:0:61548:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) for lock 0x73cfd86cfc17bc5e 00000080:00010000:1.0:1775764725.676573:0:61548:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc5e from it ffffa60b01f77cd8 00010000:00010000:1.0:1775764725.676575:0:61548:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764725.676588:0:61548:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764725.676639:0:61548:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764725.676649:0:61548:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x6:0x0] 00000008:00010000:1.0:1775764725.676677:0:61548:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764725.676681:0:61548:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5330, osc lock ffff8b2382cf6000, flags 0x400001002 00010000:00010000:1.0:1775764725.676727:0:61548:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000bb6dc114/0x73cfd86cfc17bc65 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.676738:0:61548:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000bb6dc114/0x73cfd86cfc17bc65 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1775764725.676778:0:61548:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:1.0:1775764725.676793:0:2377:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764725.676808:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098584448:192.168.202.128@tcp:101:bash.0 00010000:00010000:1.0:1775764725.684136:0:2377:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000bb6dc114/0x73cfd86cfc17bc65 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.684167:0:2377:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000bb6dc114/0x73cfd86cfc17bc65 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305196a expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.684182:0:2377:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000bb6dc114/0x73cfd86cfc17bc65 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305196a expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.684195:0:2377:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000bb6dc114/0x73cfd86cfc17bc65 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305196a expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1775764725.684213:0:2377:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000bb6dc114/0x73cfd86cfc17bc65 lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305196a expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1775764725.684226:0:2377:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1775764725.684246:0:2377:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000bb6dc114/0x73cfd86cfc17bc65 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305196a expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.684255:0:2377:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000bb6dc114/0x73cfd86cfc17bc65 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305196a expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.684263:0:2377:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000bb6dc114/0x73cfd86cfc17bc65 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e0305196a expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.684271:0:2377:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000bb6dc114/0x73cfd86cfc17bc65 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e0305196a expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1775764725.684290:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098584448:192.168.202.128@tcp:101:bash.0 00010000:00010000:2.0:1775764725.687825:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764725.689191:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119046016 00000100:00100000:1.0:1775764725.689201:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119046016 00000100:00100000:1.0:1775764725.689256:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 229 00000100:00100000:1.0:1775764725.689267:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42944:x1862022119046016:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764725.689284:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000bb6dc114/0x73cfd86cfc17bc65 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e0305196a expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764725.689530:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764725.689537:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42944:x1862022119046016:12345-192.168.202.128@tcp:104: Request processed in 272us (512us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764725.689544:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 229 00000100:00100000:2.0:1775764725.691659:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098584704:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764725.695041:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764725.695084:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119046144 00000100:00100000:1.0:1775764725.695095:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119046144 00000100:00100000:1.0:1775764725.695152:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 230 00000100:00100000:1.0:1775764725.695163:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23864fd500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:44302:x1862022119046144:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764725.695183:0:46139:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17bc6c lrc: 4/0,1 mode: --/PW res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051971 expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.695197:0:46139:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17bc6c lrc: 4/0,1 mode: --/PW res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051971 expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.695207:0:46139:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17bc6c lrc: 4/0,1 mode: --/PW res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051971 expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.695217:0:46139:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17bc6c lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e03051971 expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.695226:0:46139:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17bc6c lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e03051971 expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.695237:0:46139:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17bc6c lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000000000 nid: local remote: 0x57b8ff0e03051971 expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.695255:0:46139:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b23c2045a00) 00000100:00100000:1.0:1775764725.695304:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23864fd500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:44302:x1862022119046144:12345-192.168.202.128@tcp:105: Request processed in 143us (275us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764725.695312:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 230 00000100:00100000:2.0:1775764725.697314:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098584704:192.168.202.128@tcp:103:ldlm_bl.0 00000080:00200000:1.0:1775764725.731309:0:61550:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764725.731345:0:61550:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764725.731418:0:61550:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 61550 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.731429:0:61550:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61550 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764725.731439:0:61550:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61550 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764725.731621:0:61550:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61550:1862022098585088:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764725.731785:0:61550:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384904840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764726.159372:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200080:2.0:1775764726.165388:0:61563:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086696 arg=7fa5a0000d20 00800000:00000080:2.0:1775764726.165426:0:61563:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=c0086696 len=4128 karg=ffff8b2384974000 uarg=0000000000000000 00000002:00000080:2.0:1775764726.165439:0:61563:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=c0086696 len=4128 karg=ffff8b2384974000 uarg=0000000000000000 00000002:00000080:2.0:1775764726.165448:0:61563:0:(mdc_request.c:1738:mdc_ioc_fid2path()) path get [0x200003ab4:0x6:0x0] from 18446744073709551615 #0 00000100:00100000:2.0:1775764726.165567:0:61563:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61563:1862022098585472:192.168.202.128@tcp:53:lhsmtool_posix.0 00000100:00100000:2.0:1775764726.165689:0:61563:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764726.184805:0:61563:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61563:1862022098585472:192.168.202.128@tcp:53:lhsmtool_posix.0 00000002:00000080:2.0:1775764726.184837:0:61563:0:(mdc_request.c:1758:mdc_ioc_fid2path()) path got [0x200000007:0x1:0x0] from 18446744073709551615 #0: f22.sanity-pcc 00000080:00200000:2.0:1775764726.185033:0:61563:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000002:0x2:0x0](ffff8b2390ec5348), inode mode 4040 mask 1 00000080:00200000:2.0:1775764726.185054:0:61563:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=0x200003ab4:0x6:0x0, dir=[0x200000002:0x2:0x0](ffff8b2390ec5348), file ffff8b2386957200, open_flags 68800, mode 0 opened 0 00000080:00200000:2.0:1775764726.185062:0:61563:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=0x200003ab4:0x6:0x0, dir=[0x200000002:0x2:0x0](ffff8b2390ec5348), intent=open 00000002:00010000:2.0:1775764726.185096:0:61563:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x6:0x0]) in obj [0x200000002:0x2:0x0], intent: open flags 041504001 00010000:00010000:2.0:1775764726.185172:0:61563:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 61563 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.185183:0:61563:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61563 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764726.185194:0:61563:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764726.185203:0:61563:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5880 x1862022098585728/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764726.185213:0:61563:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61563 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764726.185236:0:61563:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61563:1862022098585728:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764726.187065:0:61563:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23913869c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764726.198933:0:61563:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61563:1862022098585728:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764726.198962:0:61563:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 4/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.198978:0:61563:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.198987:0:61563:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.199000:0:61563:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.199011:0:61563:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.199024:0:61563:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.199033:0:61563:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.199040:0:61563:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764726.199050:0:61563:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23b9ea5880 x1862022098585728/t34359739106(34359739106) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764742 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764726.199063:0:61563:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea5880 x1862022098585728/t34359739106(34359739106) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764742 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764726.199075:0:61563:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764726.199089:0:61563:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.199101:0:61563:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764726.199112:0:61563:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764726.199121:0:61563:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.199128:0:61563:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.199134:0:61563:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.199160:0:61563:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:2.0:1775764726.199168:0:61563:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000002:0x2:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764726.199254:0:61563:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0x6:0x0](ffff8b2389a61148): rc = 0 00000080:00200000:2.0:1775764726.199271:0:61563:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x6:0x0]: layout version change: 0 -> 0 00000080:00010000:2.0:1775764726.199280:0:61563:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) for lock 0x73cfd86cfc17bc5e 00000080:00010000:2.0:1775764726.199291:0:61563:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) 00000080:00200008:2.0:1775764726.199305:0:61563:0:(file.c:1057:ll_file_open()) START file 0x200003ab4:0x6:0x0:[0x200003ab4:0x6:0x0](ffff8b2389a61148), flags 1504000 00000002:00100000:2.0:1775764726.199324:0:61563:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea5880 x1862022098585728/t34359739106(34359739106) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764742 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764726.199381:0:61563:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc5e from it ffff8b238504cb40 00010000:00010000:2.0:1775764726.199385:0:61563:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764726.199394:0:61563:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764726.199434:0:61563:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148), inode mode 81a4 mask 44 00000080:00200080:2.0:1775764726.199739:0:61563:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=400866d6 arg=7fa5a0000b78 00000080:00200000:2.0:1775764726.199763:0:61563:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764726.199775:0:61563:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:2.0:1775764726.199848:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764726.199862:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098585856:192.168.202.128@tcp:1:lhsmtool_posix.0 00010000:00010000:1.0:1775764726.199994:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764726.200001:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764726.200008:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6eb200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764726.200013:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764726.200020:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764726.200029:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764726.200048:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764726.200075:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764726.200081:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764726.200103:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764726.200109:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bc7a lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305198d expref: -99 pid: 61563 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764726.200124:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764726.200126:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764726.200127:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764726.200137:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098585984:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764726.205849:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200000:2.0:1775764726.216833:0:46149:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764726.216898:0:46149:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775764726.217063:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17bc6c lrc: 2/0,0 mode: --/PW res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e03051971 expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764726.217073:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764726.217337:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17bc6c lrc: 1/0,0 mode: --/PW res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e03051971 expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764726.217347:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17bc6c lrc: 0/0,0 mode: --/PW res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e03051971 expref: -99 pid: 61548 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764726.220571:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764726.220578:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764726.220581:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764726.221893:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098585856:192.168.202.128@tcp:1:lhsmtool_posix.0 00000100:00100000:2.0:1775764726.231250:0:61563:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61563:1862022098586496:192.168.202.128@tcp:57:lhsmtool_posix.0 00000080:00200000:2.0:1775764726.231873:0:61563:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148), inode mode 81a4 mask 4 00000080:00200000:2.0:1775764726.231885:0:61563:0:(xattr.c:727:ll_xattr_get()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148), xattr lov 00000080:00200008:2.0:1775764726.232101:0:61563:0:(file.c:6226:ll_getattr_dentry()) START file 0x200003ab4:0x6:0x0:[0x200003ab4:0x6:0x0](ffff8b2389a61148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764726.232110:0:61563:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148),name=0x200003ab4:0x6:0x0 00000002:00010000:2.0:1775764726.232132:0:61563:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x6:0x0]) in obj [0x200000002:0x2:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764726.232150:0:61563:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764726.232164:0:61563:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764726.232173:0:61563:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000002:0x2:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764726.232179:0:61563:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) 00000080:00010000:2.0:1775764726.232183:0:61563:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) for lock 0x73cfd86cfc17bc5e 00000080:00010000:2.0:1775764726.232190:0:61563:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc5e from it ffffa60b01fb7cd8 00010000:00010000:2.0:1775764726.232194:0:61563:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764726.232203:0:61563:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764726.232250:0:61563:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1775764726.232259:0:61563:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x6:0x0] 00000008:00010000:2.0:1775764726.232285:0:61563:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764726.232289:0:61563:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24db0, osc lock ffff8b238a3f90a0, flags 0x400001002 00010000:00010000:2.0:1775764726.232336:0:61563:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.232348:0:61563:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1775764726.232393:0:61563:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:1.0:1775764726.238727:0:61563:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61563:1862022098586880:192.168.202.128@tcp:57:lhsmtool_posix.0 00000080:00200008:1.0:1775764726.238852:0:61563:0:(file.c:2454:do_file_read_iter()) START file 0x200003ab4:0x6:0x0:[0x200003ab4:0x6:0x0], ppos: 0, count: 14 00000080:00200000:1.0:1775764726.238948:0:61563:0:(rw.c:1928:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1775764726.238992:0:61563:0:(file.c:2058:ll_file_io_generic()) 0x200003ab4:0x6:0x0: read ppos: 0, bytes: 14 00000080:00200000:1.0:1775764726.238998:0:61563:0:(file.c:1815:ll_io_set_mirror()) 0x200003ab4:0x6:0x0: desiginated mirror: 0 00000020:00200000:1.0:1775764726.239008:0:61563:0:(cl_io.c:214:cl_io_rw_init()) header@000000006fc2d42d[0x110, 1, [0x200003ab4:0x6:0x0] hash] 00000020:00200000:1.0:1775764726.239011:0:61563:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 14) 2048 0 00000080:00200000:1.0:1775764726.239015:0:61563:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764726.239035:0:61563:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 14) 14, 14 00020000:00200000:1.0:1775764726.239039:0:61563:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764726.239069:0:61563:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 0] 00000008:00010000:1.0:1775764726.239083:0:61563:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 1 00000008:00010000:1.0:1775764726.239087:0:61563:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c55b0, osc lock ffff8b2382cf6be0, flags 0x40000 00010000:00010000:1.0:1775764726.239107:0:61563:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764726.239123:0:61563:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764726.239136:0:61563:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764726.239151:0:61563:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764726.239158:0:61563:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764726.239168:0:61563:0:(vvp_io.c:826:vvp_io_read_start()) 0x200003ab4:0x6:0x0: read [0, 14) 00000080:00200000:1.0:1775764726.239180:0:61563:0:(vvp_io.c:178:vvp_prep_size()) [0x200003ab4:0x6:0x0] updating i_size 14 00000080:00200000:1.0:1775764726.239183:0:61563:0:(vvp_io.c:885:vvp_io_read_start()) count:14 iocb pos:0 00000008:00010000:1.0:1775764726.239222:0:61563:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b238a0cef00 00000008:00010000:1.0:1775764726.239269:0:30878:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b238a0cef00 00010000:00010000:1.0:1775764726.239288:0:30878:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00100000:1.0:1775764726.239323:0:30878:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23864fce00 - object 0x240000400:281 offset 0<>4096 00000100:00100000:1.0:1775764726.239773:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098587008:192.168.202.128@tcp:3:lhsmtool_posix.0 00000008:00010000:1.0:1775764726.242791:0:2376:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8b238a0cef00 00000020:00200000:1.0:1775764726.242817:0:2376:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1775764726.242894:0:2376:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8b238a0cef00 00000100:00100000:1.0:1775764726.242928:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098587008:192.168.202.128@tcp:3:lhsmtool_posix.0 00000100:00100000:2.0:1775764726.282802:0:61563:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61563:1862022098587136:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764726.282835:0:61563:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bc88 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030519b0 expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.282851:0:61563:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bc88 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030519b0 expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.282863:0:61563:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bc88 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030519b0 expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764726.282877:0:61563:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=80 disposition=0, status=0 req@ffff8b2382dae680 x1862022098587136/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 568/1216 e 0 to 0 dl 1775764742 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764726.282889:0:61563:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bc88 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030519b0 expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.282903:0:61563:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/6 (0 0) 00000080:00200000:2.0:1775764726.282909:0:61563:0:(namei.c:953:ll_intent_lock()) intent lock 128 on i1 [0x200003ab4:0x6:0x0] suppgids 0 -1: rc 0 00000080:00010000:2.0:1775764726.282941:0:61563:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) for lock 0x73cfd86cfc17bc88 00000080:00010000:2.0:1775764726.282951:0:61563:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc88 from it ffffa60b01fb7d08 00010000:00010000:2.0:1775764726.282955:0:61563:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bc88 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030519b0 expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.282971:0:61563:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bc88 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030519b0 expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764726.283175:0:61563:0:(xattr.c:727:ll_xattr_get()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148), xattr hsm 00000080:00200000:2.0:1775764726.283353:0:61563:0:(xattr.c:727:ll_xattr_get()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148), xattr version 00000080:00200000:2.0:1775764726.283417:0:61563:0:(xattr.c:727:ll_xattr_get()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148), xattr link 00000080:00200000:2.0:1775764726.283655:0:61563:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148), inode mode 81a4 mask 4 00000080:00200000:2.0:1775764726.283730:0:61563:0:(xattr.c:727:ll_xattr_get()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148), xattr lov 00000080:00200000:2.0:1775764726.283821:0:61563:0:(xattr.c:727:ll_xattr_get()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148), xattr lma 00000080:00200000:2.0:1775764726.283885:0:61563:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148), inode mode 81a4 mask 4 00000080:00200000:2.0:1775764726.283890:0:61563:0:(xattr.c:727:ll_xattr_get()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148), xattr lov 00000080:00200080:2.0:1775764726.284094:0:61563:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086696 arg=7fa5a0000d70 00800000:00000080:2.0:1775764726.284113:0:61563:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=c0086696 len=4082 karg=ffff8b2383d2e000 uarg=0000000000000000 00000002:00000080:2.0:1775764726.284122:0:61563:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=c0086696 len=4082 karg=ffff8b2383d2e000 uarg=0000000000000000 00000002:00000080:2.0:1775764726.284128:0:61563:0:(mdc_request.c:1738:mdc_ioc_fid2path()) path get [0x200003ab4:0x6:0x0] from 18446744073709551615 #0 00000100:00100000:2.0:1775764726.284178:0:61563:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61563:1862022098587264:192.168.202.128@tcp:53:lhsmtool_posix.0 00000100:00100000:2.0:1775764726.284280:0:61563:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764726.289842:0:61563:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61563:1862022098587264:192.168.202.128@tcp:53:lhsmtool_posix.0 00000002:00000080:2.0:1775764726.289878:0:61563:0:(mdc_request.c:1758:mdc_ioc_fid2path()) path got [0x200000007:0x1:0x0] from 18446744073709551615 #0: f22.sanity-pcc 00000080:00200080:2.0:1775764726.290311:0:61563:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=400866d7 arg=7fa5a0000b78 00000080:00200000:2.0:1775764726.290375:0:61563:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764726.290387:0:61563:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00800000:00000080:2.0:1775764726.304230:0:61563:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b01fb7cf0 uarg=0000000000000000 00000002:00000080:2.0:1775764726.304242:0:61563:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b01fb7cf0 uarg=0000000000000000 00000020:00100000:2.0:1775764726.304273:0:61563:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 57, max 7 00000100:00100000:2.0:1775764726.304281:0:61563:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5180 x1862022098587648/t0(0) o57->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764726.304304:0:61563:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61563:1862022098587648:192.168.202.128@tcp:57:lhsmtool_posix.0 00000100:00100000:2.0:1775764726.304404:0:61563:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386180 going to sleep for 11 seconds 00000100:00100000:1.0:1775764726.313353:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119046528 00000100:00100000:1.0:1775764726.313364:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119046528 00000100:00100000:1.0:1775764726.313421:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 233 00000100:00100000:1.0:1775764726.313432:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dadf80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119046528:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764726.313489:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bc88 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e030519b0 expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764726.313568:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764726.313582:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dadf80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119046528:12345-192.168.202.128@tcp:104: Request processed in 152us (463us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764726.313596:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 233 00010000:00010000:2.0:1775764726.314085:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764726.314093:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bc88 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e030519b0 expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764726.314102:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23c2044a00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764726.314107:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bc88 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e030519b0 expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764726.314116:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bc88 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e030519b0 expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764726.314124:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bc88 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x57b8ff0e030519b0 expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764726.314194:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bc88 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030519b0 expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764726.314202:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764726.314230:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bc88 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030519b0 expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764726.314237:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bc88 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030519b0 expref: -99 pid: 61563 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764726.314258:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764726.314260:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764726.314262:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764726.324039:0:61563:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61563:1862022098587648:192.168.202.128@tcp:57:lhsmtool_posix.0 00000080:00200008:2.0:1775764726.324091:0:61563:0:(file.c:447:ll_file_release()) START file 0x200003ab4:0x6:0x0:[0x200003ab4:0x6:0x0](ffff8b2389a61148), flags 1504000 00000002:00100000:2.0:1775764726.324162:0:61563:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea5880 x1862022098585728/t34359739106(34359739106) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764742 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764726.324176:0:61563:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764726.324181:0:61563:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4380 x1862022098588032/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764726.324206:0:61563:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61563:1862022098588032:192.168.202.128@tcp:35:lhsmtool_posix.0 00000100:00100000:2.0:1775764726.324319:0:61563:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764726.330137:0:61563:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61563:1862022098588032:192.168.202.128@tcp:35:lhsmtool_posix.0 00000002:00100000:2.0:1775764726.330174:0:61563:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea5880 x1862022098585728/t34359739106(34359739106) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764742 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764726.623883:0:61564:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764726.623910:0:61564:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764726.623931:0:61564:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764726.623947:0:61564:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764726.623960:0:61564:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764726.623970:0:61564:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764726.623974:0:61564:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bc73 00000080:00010000:1.0:1775764726.623981:0:61564:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc73 from it ffffa60b01f47b58 00010000:00010000:1.0:1775764726.623985:0:61564:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764726.623995:0:61564:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764726.624012:0:61564:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764726.624019:0:61564:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f22.sanity-pcc, flags=8 00000080:00200008:1.0:1775764726.624031:0:61564:0:(file.c:6226:ll_getattr_dentry()) START file f22.sanity-pcc:[0x200003ab4:0x6:0x0](ffff8b2389a61148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764726.624035:0:61564:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148),name=f22.sanity-pcc 00000002:00010000:1.0:1775764726.624046:0:61564:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764726.624052:0:61564:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764726.624059:0:61564:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 1 00000080:00200000:1.0:1775764726.624065:0:61564:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764726.624069:0:61564:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) 00000080:00010000:1.0:1775764726.624071:0:61564:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) for lock 0x73cfd86cfc17bc5e 00000080:00010000:1.0:1775764726.624074:0:61564:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc5e from it ffffa60b01f47cb8 00010000:00010000:1.0:1775764726.624077:0:61564:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764726.624085:0:61564:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00000080:00200000:1.0:1775764726.624155:0:61564:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764726.624164:0:61564:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x6:0x0] 00000008:00010000:1.0:1775764726.624193:0:61564:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764726.624197:0:61564:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5630, osc lock ffff8b2382cf6130, flags 0x400001002 00010000:00010000:1.0:1775764726.624212:0:61564:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764726.624224:0:61564:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1775764726.624235:0:61564:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764726.624238:0:61564:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764726.624246:0:61564:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00000020:00200000:1.0:1775764726.624255:0:61564:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764726.624263:0:61564:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x6:0x0] updating i_size 14 i_blocks 1 00000020:00010000:1.0:1775764726.624267:0:61564:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f186d8 (ffff8b2387cbacf8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764726.624269:0:61564:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f186d8 (ffff8b2387cbacf8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764726.624272:0:61564:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5630 (ffff8b2387cba758/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764726.624275:0:61564:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f186d8 (ffff8b2387cbacf8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764726.624278:0:61564:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5630 (ffff8b2387cbacf8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764726.624297:0:61564:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764726.624325:0:61564:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764726.624332:0:61564:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764726.624338:0:61564:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764726.624346:0:61564:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764726.624352:0:61564:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764726.624355:0:61564:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764726.624358:0:61564:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bc73 00000080:00010000:1.0:1775764726.624361:0:61564:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc73 from it ffffa60b01f47b98 00010000:00010000:1.0:1775764726.624363:0:61564:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764726.624370:0:61564:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:1.0:1775764726.624378:0:61564:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764726.624383:0:61564:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f22.sanity-pcc, flags=8 00000080:00200008:1.0:1775764726.624433:0:61564:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764726.624437:0:61564:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764726.624443:0:61564:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764726.624450:0:61564:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764726.624489:0:61564:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:1.0:1775764726.624495:0:61564:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764726.624499:0:61564:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764726.624503:0:61564:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bc73 00000080:00010000:1.0:1775764726.624507:0:61564:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc73 from it ffffa60b01f47cb8 00010000:00010000:1.0:1775764726.624509:0:61564:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764726.624516:0:61564:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:1.0:1775764726.624531:0:61564:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764726.624537:0:61564:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764726.624543:0:61564:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:1.0:1775764726.624552:0:61564:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:1.0:1775764726.624558:0:61564:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764726.624562:0:61564:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764726.624565:0:61564:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bc73 00000080:00010000:1.0:1775764726.624568:0:61564:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc73 from it ffffa60b01f47b58 00010000:00010000:1.0:1775764726.624570:0:61564:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:1.0:1775764726.624577:0:61564:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:1.0:1775764726.624925:0:61564:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764726.624931:0:61564:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f22.sanity-pcc, flags=8 00000080:00200008:1.0:1775764726.624936:0:61564:0:(file.c:6226:ll_getattr_dentry()) START file f22.sanity-pcc:[0x200003ab4:0x6:0x0](ffff8b2389a61148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764726.624940:0:61564:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148),name=f22.sanity-pcc 00000002:00010000:1.0:1775764726.624949:0:61564:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764726.624955:0:61564:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764726.624963:0:61564:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00000080:00200000:1.0:1775764726.624969:0:61564:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764726.624972:0:61564:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) 00000080:00010000:1.0:1775764726.624975:0:61564:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) for lock 0x73cfd86cfc17bc5e 00000080:00010000:1.0:1775764726.624978:0:61564:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc5e from it ffffa60b01f47cb8 00010000:00010000:1.0:1775764726.624981:0:61564:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764726.624988:0:61564:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00000080:00200000:1.0:1775764726.625030:0:61564:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764726.625035:0:61564:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x6:0x0] 00000008:00010000:1.0:1775764726.625051:0:61564:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764726.625054:0:61564:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5630, osc lock ffff8b2382cf6130, flags 0x400001002 00010000:00010000:1.0:1775764726.625064:0:61564:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00010000:00010000:1.0:1775764726.625073:0:61564:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00000020:00200000:1.0:1775764726.625082:0:61564:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764726.625085:0:61564:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00010000:00010000:1.0:1775764726.625093:0:61564:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000020:00200000:1.0:1775764726.625101:0:61564:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764726.625106:0:61564:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x6:0x0] updating i_size 14 i_blocks 1 00000020:00010000:1.0:1775764726.625110:0:61564:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f186d8 (ffff8b2387cbacf8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764726.625112:0:61564:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f186d8 (ffff8b2387cbacf8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764726.625115:0:61564:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5630 (ffff8b2387cba758/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764726.625117:0:61564:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f186d8 (ffff8b2387cbacf8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764726.625120:0:61564:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5630 (ffff8b2387cbacf8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764726.625134:0:61564:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x6:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1775764726.626077:0:61564:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764726.626084:0:61564:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764726.626095:0:61564:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764726.626106:0:61564:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1775764726.626117:0:61564:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:1.0:1775764726.626124:0:61564:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764726.626130:0:61564:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764726.626133:0:61564:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bc73 00000080:00010000:1.0:1775764726.626139:0:61564:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc73 from it ffffa60b01f47cb8 00010000:00010000:1.0:1775764726.626142:0:61564:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1775764726.626149:0:61564:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1775764726.626209:0:61564:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764726.626216:0:61564:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764726.626222:0:61564:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764726.626230:0:61564:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1775764726.626235:0:61564:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764726.626239:0:61564:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764726.626241:0:61564:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bc73 00000080:00010000:1.0:1775764726.626244:0:61564:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc73 from it ffffa60b01f47bb0 00010000:00010000:1.0:1775764726.626246:0:61564:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764726.626252:0:61564:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764726.626260:0:61564:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764726.626266:0:61564:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:1.0:1775764726.626687:0:61564:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=401866d9 arg=125e2a0 00000002:00010000:1.0:1775764726.626712:0:61564:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x6:0x0]) in obj [0x200003ab4:0x6:0x0], intent: open flags 03004040000002 00010000:00010000:1.0:1775764726.626775:0:61564:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bc8f lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 61564 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764726.626783:0:61564:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x81000000001000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bc8f lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61564 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764726.626796:0:61564:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764726.626807:0:61564:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fce00 x1862022098588288/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764726.626816:0:61564:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bc8f lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x81000000000000 pid: 61564 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764726.626838:0:61564:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61564:1862022098588288:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764726.627031:0:61564:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384904f00 going to sleep for 11 seconds 00000100:00100000:1.0:1775764726.634823:0:61564:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61564:1862022098588288:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764726.634850:0:61564:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PW ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bc8f lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e030519cc expref: -99 pid: 61564 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764726.634864:0:61564:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bc8f lrc: 4/1,0 mode: --/PW res: [0x200003ab4:0x6:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e030519cc expref: -99 pid: 61564 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764726.634878:0:61564:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bc8f lrc: 4/1,0 mode: PW/PW res: [0x200003ab4:0x6:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e030519cc expref: -99 pid: 61564 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764726.634888:0:61564:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bc8f lrc: 4/1,0 mode: PW/PW res: [0x200003ab4:0x6:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e030519cc expref: -99 pid: 61564 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764726.634900:0:61564:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bc8f lrc: 5/1,1 mode: PW/PW res: [0x200003ab4:0x6:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e030519cc expref: -99 pid: 61564 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764726.634907:0:61564:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bc8f lrc: 5/1,1 mode: PW/PW res: [0x200003ab4:0x6:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e030519cc expref: -99 pid: 61564 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764726.634914:0:61564:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bc8f lrc: 4/0,1 mode: PW/PW res: [0x200003ab4:0x6:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e030519cc expref: -99 pid: 61564 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764726.634925:0:61564:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=600002b, status=0 req@ffff8b23864fce00 x1862022098588288/t34359739109(34359739109) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764742 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764726.634939:0:61564:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fce00 x1862022098588288/t34359739109(34359739109) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764742 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764726.634956:0:61564:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bc8f lrc: 3/0,1 mode: PW/PW res: [0x200003ab4:0x6:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e030519cc expref: -99 pid: 61564 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764726.634969:0:61564:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 2 res 8589949620/6 (0 0) 00000080:00200000:1.0:1775764726.634975:0:61564:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab4:0x6:0x0] suppgids 0 0: rc 0 00000002:00100000:1.0:1775764726.634993:0:61564:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fce00 x1862022098588288/t34359739109(34359739109) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764742 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1775764726.635001:0:61564:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) for lock 0x73cfd86cfc17bc8f 00000080:00010000:1.0:1775764726.635007:0:61564:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc8f from it ffffa60b01f47b80 00010000:00010000:1.0:1775764726.635011:0:61564:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bc8f lrc: 3/0,1 mode: PW/PW res: [0x200003ab4:0x6:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e030519cc expref: -99 pid: 61564 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764726.635017:0:61564:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bc8f lrc: 2/0,0 mode: PW/PW res: [0x200003ab4:0x6:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e030519cc expref: -99 pid: 61564 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1775764726.635089:0:61564:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764726.635101:0:61564:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:1.0:1775764726.635175:0:2376:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764726.635190:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098588416:192.168.202.128@tcp:1:lfs.0 00000100:00100000:1.0:1775764726.639215:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119046784 00000100:00100000:1.0:1775764726.639226:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119046784 00000100:00100000:1.0:1775764726.639322:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 234 00000100:00100000:1.0:1775764726.639347:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dae680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42946:x1862022119046784:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764726.639367:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1775764726.639527:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764726.639534:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dae680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42946:x1862022119046784:12345-192.168.202.128@tcp:104: Request processed in 204us (733us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764726.639542:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 234 00010000:00010000:2.0:1775764726.639991:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764726.639998:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1775764726.640009:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6eaa00 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:2.0:1775764726.640015:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000080:00200000:2.0:1775764726.640076:0:46149:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764726.640142:0:46149:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775764726.640210:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 2/0,0 mode: --/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1775764726.640219:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764726.640626:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1775764726.640637:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000007e745dfa/0x73cfd86cfc17bc81 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e030519a2 expref: -99 pid: 61563 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1775764726.640660:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764726.640662:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764726.640663:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764726.652381:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098588416:192.168.202.128@tcp:1:lfs.0 00000080:00200000:2.0:1775764726.652628:0:61564:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x6:0x0] updating i_size 14 i_blocks 1 00010000:00010000:2.0:1775764726.652699:0:61564:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bc8f lrc: 2/0,0 mode: PW/PW res: [0x200003ab4:0x6:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x57b8ff0e030519cc expref: -99 pid: 61564 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.652722:0:61564:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bc8f lrc: 0/0,0 mode: --/PW res: [0x200003ab4:0x6:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x57b8ff0e030519cc expref: -99 pid: 61564 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764726.652746:0:61564:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fce00 x1862022098588288/t34359739109(34359739109) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764742 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764726.652759:0:61564:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764726.652765:0:61564:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5880 x1862022098588800/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 504/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764726.652791:0:61564:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61564:1862022098588800:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764726.652891:0:61564:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386180 going to sleep for 11 seconds 00000100:00100000:1.0:1775764726.660894:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119046912 00000100:00100000:1.0:1775764726.660911:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119046912 00000100:00100000:1.0:1775764726.660993:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 235 00000100:00100000:1.0:1775764726.661002:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42328:x1862022119046912:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764726.661025:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00010000:00010000:1.0:1775764726.661127:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764726.661133:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42328:x1862022119046912:12345-192.168.202.128@tcp:104: Request processed in 132us (4762us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764726.661139:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 235 00010000:00010000:2.0:1775764726.661196:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764726.661203:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1775764726.661212:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8741a00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764726.661217:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1775764726.661226:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1775764726.661235:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00000080:00200000:2.0:1775764726.661285:0:46150:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x6:0x0]: losing layout lock 00010000:00010000:2.0:1775764726.661371:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1775764726.661379:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764726.661415:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1775764726.661423:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17bc5e lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e0305195c expref: -99 pid: 61548 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1775764726.661448:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764726.661449:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764726.661494:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764726.664588:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098588928:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764726.668345:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098588928:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764726.675776:0:61564:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61564:1862022098588800:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764726.675815:0:61564:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fce00 x1862022098588288/t34359739109(34359739109) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764742 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1775764726.677042:0:61564:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:2.0:1775764726.697873:0:61567:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764726.697901:0:61567:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764726.697918:0:61567:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764726.698018:0:61567:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:2.0:1775764726.698028:0:61567:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764726.698036:0:61567:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764726.698039:0:61567:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bc73 00000080:00010000:2.0:1775764726.698046:0:61567:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc73 from it ffffa60b01fbfbf8 00010000:00010000:2.0:1775764726.698050:0:61567:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764726.698064:0:61567:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17bc73 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305197f expref: -99 pid: 61550 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:2.0:1775764726.698079:0:61567:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764726.698094:0:61567:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f22.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b23a0ec2780, open_flags 8800, mode 0 opened 0 00000080:00200000:2.0:1775764726.698100:0:61567:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f22.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open 00000002:00010000:2.0:1775764726.698112:0:61567:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f22.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104001 00010000:00010000:2.0:1775764726.698159:0:61567:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc96 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 61567 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.698167:0:61567:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc96 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61567 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764726.698178:0:61567:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764726.698190:0:61567:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5880 x1862022098589184/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764726.698200:0:61567:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc96 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61567 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764726.698224:0:61567:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61567:1862022098589184:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764726.698824:0:61567:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764726.701330:0:61567:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61567:1862022098589184:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764726.701357:0:61567:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc96 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030519fd expref: -99 pid: 61567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.701386:0:61567:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc96 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030519fd expref: -99 pid: 61567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.701395:0:61567:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc96 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030519fd expref: -99 pid: 61567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.701405:0:61567:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc96 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030519fd expref: -99 pid: 61567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.701416:0:61567:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc96 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030519fd expref: -99 pid: 61567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.701429:0:61567:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc96 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030519fd expref: -99 pid: 61567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.701438:0:61567:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc96 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030519fd expref: -99 pid: 61567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.701445:0:61567:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc96 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030519fd expref: -99 pid: 61567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764726.701495:0:61567:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23b9ea5880 x1862022098589184/t34359739118(34359739118) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764742 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764726.701508:0:61567:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea5880 x1862022098589184/t34359739118(34359739118) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764742 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764726.701520:0:61567:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc96 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030519fd expref: -99 pid: 61567 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764726.701532:0:61567:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc96 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030519fd expref: -99 pid: 61567 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.701542:0:61567:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/6 (0 0) 00000080:00200000:2.0:1775764726.701548:0:61567:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764726.701642:0:61567:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764726.701655:0:61567:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764726.701719:0:61567:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764726.701787:0:61567:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0x6:0x0], updating i_size 14 00000080:00200000:2.0:1775764726.701791:0:61567:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0x6:0x0](ffff8b2389a61148): rc = 0 00000080:00200000:2.0:1775764726.701808:0:61567:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x6:0x0]: layout version change: 4294967294 -> 1 00000080:00010000:2.0:1775764726.701819:0:61567:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) for lock 0x73cfd86cfc17bc96 00000080:00010000:2.0:1775764726.701832:0:61567:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) 00000080:00200008:2.0:1775764726.701848:0:61567:0:(file.c:1057:ll_file_open()) START file f22.sanity-pcc:[0x200003ab4:0x6:0x0](ffff8b2389a61148), flags 104000 00000002:00100000:2.0:1775764726.701869:0:61567:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea5880 x1862022098589184/t34359739118(34359739118) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764742 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764726.701918:0:61567:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bc96 from it ffff8b238504c480 00010000:00010000:2.0:1775764726.702080:0:61567:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc96 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030519fd expref: -99 pid: 61567 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764726.702091:0:61567:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17bc96 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030519fd expref: -99 pid: 61567 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764726.702133:0:61567:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148), inode mode 81a4 mask 44 00000080:00200080:2.0:1775764726.702166:0:61567:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148) cmd=802066d3 arg=7ffcf8d1d440 00800000:00000080:2.0:1775764726.702177:0:61567:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=802066d3 len=384 karg=ffff8b238a3c3800 uarg=0000000000000000 00000002:00000080:2.0:1775764726.702183:0:61567:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=802066d3 len=384 karg=ffff8b238a3c3800 uarg=0000000000000000 00000100:00100000:2.0:1775764726.702220:0:61567:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61567:1862022098589312:192.168.202.128@tcp:54:lfs.0 00000100:00100000:2.0:1775764726.702304:0:61567:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764726.708829:0:61567:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61567:1862022098589312:192.168.202.128@tcp:54:lfs.0 00000080:00200008:2.0:1775764726.708897:0:61567:0:(file.c:447:ll_file_release()) START file f22.sanity-pcc:[0x200003ab4:0x6:0x0](ffff8b2389a61148), flags 104000 00000002:00100000:2.0:1775764726.708971:0:61567:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea5880 x1862022098589184/t34359739118(34359739118) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764742 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764726.708983:0:61567:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764726.708988:0:61567:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5f80 x1862022098589568/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764726.709009:0:61567:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61567:1862022098589568:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764726.709080:0:61567:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386180 going to sleep for 11 seconds 00010000:00010000:2.0:1775764726.711707:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764726.712296:0:61567:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61567:1862022098589568:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764726.712326:0:61567:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea5880 x1862022098589184/t34359739118(34359739118) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764742 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764727.379773:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119047936 00000100:00100000:1.0:1775764727.379785:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119047936 00000100:00100000:1.0:1775764727.379849:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 236 00000100:00100000:1.0:1775764727.379861:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dac700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42356:x1862022119047936:12345-192.168.202.128@tcp:107:hsm_cdtr.0 00000100:00100000:1.0:1775764727.379958:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dac700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42356:x1862022119047936:12345-192.168.202.128@tcp:107:hsm_cdtr.0 Request processed in 100us (522us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764727.379966:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 236 00000100:00100000:1.0:1775764727.509293:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:2.0:1775764727.530190:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098593664:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764727.543722:0:2379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17bcdc lrc: 6/2,0 mode: --/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764727.543752:0:2379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17bcdc lrc: 6/2,0 mode: --/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051acf expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764727.543767:0:2379:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17bcdc lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051acf expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764727.543783:0:2379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17bcdc lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051acf expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1775764727.543799:0:2379:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17bcdc lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051acf expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1775764727.543812:0:2379:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:2.0:1775764727.543832:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17bcdc lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051acf expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764727.543840:0:2379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17bcdc lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051acf expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764727.543847:0:2379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17bcdc lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051acf expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764727.543856:0:2379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17bcdc lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051acf expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:2.0:1775764727.543878:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098593664:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764727.589697:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764727.591509:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764727.591519:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17bcdc lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e03051acf expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764727.591538:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5ec00 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:2.0:1775764727.591545:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17bcdc lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x57b8ff0e03051acf expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764727.591609:0:46149:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x8:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764727.591624:0:46149:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x8:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775764727.591694:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17bcdc lrc: 2/0,0 mode: --/PR res: [0x280000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e03051acf expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764727.591702:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764727.591728:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17bcdc lrc: 1/0,0 mode: --/PR res: [0x280000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e03051acf expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764727.591737:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17bcdc lrc: 0/0,0 mode: --/PR res: [0x280000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e03051acf expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764727.591756:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764727.591758:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764727.591759:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764727.591769:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764727.591784:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098594176:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764727.608797:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119048320 00000100:00100000:1.0:1775764727.608809:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119048320 00000100:00100000:1.0:1775764727.608857:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 239 00000100:00100000:1.0:1775764727.608866:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dadf80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42310:x1862022119048320:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764727.608880:0:46140:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000009d0e3d49/0x73cfd86cfc17bce3 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03051ad6 expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764727.608891:0:46140:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000009d0e3d49/0x73cfd86cfc17bce3 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03051ad6 expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764727.608900:0:46140:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000009d0e3d49/0x73cfd86cfc17bce3 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e03051ad6 expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764727.608909:0:46140:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000009d0e3d49/0x73cfd86cfc17bce3 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x57b8ff0e03051ad6 expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764727.608930:0:46140:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000009d0e3d49/0x73cfd86cfc17bce3 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x57b8ff0e03051ad6 expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764727.608943:0:46140:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000009d0e3d49/0x73cfd86cfc17bce3 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x57b8ff0e03051ad6 expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764727.608969:0:46140:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b23b9c5fa00) 00000100:00100000:1.0:1775764727.609028:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dadf80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42310:x1862022119048320:12345-192.168.202.128@tcp:105: Request processed in 162us (704us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764727.609035:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 239 00000100:00100000:2.0:1775764727.615169:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098594176:192.168.202.128@tcp:103:ldlm_bl.0 00000008:00010000:2.0:1775764727.625133:0:2378:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 000000007376cb7b ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000009d0e3d49/0x73cfd86cfc17bce3 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03051ad6 expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:2.0:1775764727.625270:0:2378:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000007376cb7b ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000009d0e3d49/0x73cfd86cfc17bce3 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03051ad6 expref: -99 pid: 61578 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1775764727.625305:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098594304:192.168.202.128@tcp:4:lhsmtool_posix.0 00000100:00100000:2.0:1775764727.633880:0:61578:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61578:1862022098594432:192.168.202.128@tcp:44:lhsmtool_posix.0 00000100:00100000:2.0:1775764727.633992:0:61578:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764727.777325:0:61578:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61578:1862022098594432:192.168.202.128@tcp:44:lhsmtool_posix.0 00000080:00200000:2.0:1775764727.777432:0:61578:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x8:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764727.777483:0:61578:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:2.0:1775764727.777600:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764727.777618:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098594560:192.168.202.128@tcp:16:lhsmtool_posix.0 00000100:00100000:2.0:1775764727.879272:0:2379:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b2389398000: committing for last_committed 34359738447 gen 1 00000100:00100000:2.0:1775764727.879337:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098594560:192.168.202.128@tcp:16:lhsmtool_posix.0 00000100:00100000:1.0:1775764727.911090:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119048576 00000100:00100000:1.0:1775764727.911101:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119048576 00000100:00100000:1.0:1775764727.911157:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 240 00000100:00100000:1.0:1775764727.911167:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:44493:x1862022119048576:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764727.911188:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17bcc0 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x8:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03051a97 expref: -99 pid: 61578 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764727.911273:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764727.911280:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:44493:x1862022119048576:12345-192.168.202.128@tcp:104: Request processed in 115us (445us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764727.911288:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 240 00010000:00010000:2.0:1775764727.911498:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764727.911505:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17bcc0 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x8:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03051a97 expref: -99 pid: 61578 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764727.911514:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23c2044800 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764727.911519:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17bcc0 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x8:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03051a97 expref: -99 pid: 61578 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764727.911527:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17bcc0 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x8:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03051a97 expref: -99 pid: 61578 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764727.911536:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17bcc0 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x8:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x57b8ff0e03051a97 expref: -99 pid: 61578 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764727.911621:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17bcc0 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x8:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03051a97 expref: -99 pid: 61578 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764727.911629:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764727.911660:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17bcc0 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x8:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03051a97 expref: -99 pid: 61578 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764727.911667:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17bcc0 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x8:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03051a97 expref: -99 pid: 61578 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764727.911681:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764727.911683:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764727.911684:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764727.911693:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764727.911710:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098595456:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764727.929081:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098595456:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764727.953697:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:2.0:1775764727.954648:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098595712:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764727.961364:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098595712:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764727.965872:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764727.966872:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764727.966881:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bcea lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x8:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03051aeb expref: -99 pid: 61578 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764727.966896:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5e200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764727.966901:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bcea lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x8:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03051aeb expref: -99 pid: 61578 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764727.966909:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bcea lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x8:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03051aeb expref: -99 pid: 61578 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764727.967000:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bcea lrc: 3/0,0 mode: CR/CR res: [0x200003ab4:0x8:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03051aeb expref: -99 pid: 61578 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764727.967057:0:46149:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x8:0x0]: losing layout lock 00010000:00010000:2.0:1775764727.967122:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bcea lrc: 2/0,0 mode: --/CR res: [0x200003ab4:0x8:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03051aeb expref: -99 pid: 61578 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764727.967130:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764727.967788:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bcea lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x8:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03051aeb expref: -99 pid: 61578 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764727.967799:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17bcea lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x8:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03051aeb expref: -99 pid: 61578 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764727.967830:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764727.967831:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764727.967833:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764727.979088:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098596096:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764727.981224:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764728.012508:0:61578:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61578:1862022098596480:192.168.202.128@tcp:35:lhsmtool_posix.0 00000002:00100000:1.0:1775764728.012534:0:61578:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b2382dac000 x1862022098591488/t34359739123(34359739123) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 552/608 e 0 to 0 dl 1775764743 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764728.013312:0:61577:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x6:0x0]: losing layout lock 00010000:00010000:2.0:1775764728.013364:0:61577:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17bca4 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051a2e expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764728.013384:0:61577:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764728.013387:0:61577:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17bca4 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051a2e expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764728.013416:0:61577:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x6:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 61577 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.013423:0:61577:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x6:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61577 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764728.013445:0:61577:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764728.013494:0:61577:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864f1f80 x1862022098596736/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/880 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764728.013505:0:61577:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x6:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61577 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764728.013527:0:61577:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61577:1862022098596736:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764728.013625:0:61577:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764728.022366:0:61577:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61577:1862022098596736:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764728.022397:0:61577:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 4/0,1 mode: --/EX res: [0x200003ab4:0x6:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.022409:0:61577:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 4/0,1 mode: --/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.022420:0:61577:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.022434:0:61577:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.022499:0:61577:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.022508:0:61577:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.022516:0:61577:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764728.022525:0:61577:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23864f1f80 x1862022098596736/t34359739136(34359739136) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/600 e 0 to 0 dl 1775764744 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764728.022544:0:61577:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764728.022555:0:61577:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.022578:0:61577:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/6 (0 0) 00000080:00200000:2.0:1775764728.022586:0:61577:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x6:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764728.022597:0:61577:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) for lock 0x73cfd86cfc17bcf1 00000080:00010000:2.0:1775764728.022605:0:61577:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bcf1 from it ffffa60b01fdfc10 00010000:00010000:2.0:1775764728.022608:0:61577:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.022621:0:61577:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764728.022645:0:61577:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764728.022657:0:61577:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764728.022665:0:61577:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x6:0x0](00000000056824b7) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764728.022753:0:61577:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764728.022761:0:61577:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764728.022792:0:61577:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764728.022819:0:61577:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764728.022868:0:61577:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764728.022892:0:61577:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764728.022900:0:61577:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764728.022912:0:61577:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x6:0x0](00000000056824b7) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764728.022944:0:61577:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764728.022949:0:61577:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764728.022976:0:61577:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764728.023038:0:61577:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x6:0x0]: layout version change: 4294967294 -> 3 00010000:00010000:2.0:1775764728.023049:0:61577:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764728.023057:0:61577:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:2.0:1775764728.023442:0:61577:0:(file.c:2454:do_file_read_iter()) START file f22.sanity-pcc:[0x200003ab4:0x6:0x0], ppos: 0, count: 1048576 00000080:00200000:2.0:1775764728.023502:0:61577:0:(rw.c:1928:ll_readpage()) fast read pgno: 0 00000080:00200000:2.0:1775764728.023525:0:61577:0:(file.c:2058:ll_file_io_generic()) f22.sanity-pcc: read ppos: 0, bytes: 1048576 00000080:00200000:2.0:1775764728.023530:0:61577:0:(file.c:1815:ll_io_set_mirror()) f22.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764728.023536:0:61577:0:(cl_io.c:214:cl_io_rw_init()) header@000000006fc2d42d[0x110, 1, [0x200003ab4:0x6:0x0] hash] 00000020:00200000:2.0:1775764728.023538:0:61577:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 1048576) 0 0 00000080:00200000:2.0:1775764728.023541:0:61577:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764728.023550:0:61577:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x6:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764728.023553:0:61577:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764728.023559:0:61577:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:2.0:1775764728.023561:0:61577:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764728.023597:0:61577:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:2.0:1775764728.023610:0:61577:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764728.023615:0:61577:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24f30, osc lock ffff8b238a3f8390, flags 0x400000000000000 00010000:00010000:2.0:1775764728.023661:0:61577:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.023672:0:61577:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000000000000 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.023684:0:61577:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff8b23b9ea5f80 x1862022098596864/t0(0) o101->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764728.023695:0:61577:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764728.023714:0:61577:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61577:1862022098596864:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764728.023813:0:61577:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764728.028521:0:61577:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61577:1862022098596864:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764728.028550:0:61577:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 4/1,0 mode: --/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b31 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.028582:0:61577:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b31 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.028595:0:61577:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b31 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.028611:0:61577:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b31 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1775764728.028624:0:61577:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 14, setting rss=14; kms=14, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 4/2,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051b31 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.028637:0:61577:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051b31 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.028645:0:61577:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051b31 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764728.028671:0:61577:0:(vvp_io.c:826:vvp_io_read_start()) f22.sanity-pcc: read [0, 1048576) 00000001:00010000:2.0:1775764728.028694:0:61577:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x6:0x0] 00000008:00010000:2.0:1775764728.028708:0:61577:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764728.028712:0:61577:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24430, osc lock ffff8b238a3f9c80, flags 0x400000400001002 00010000:00010000:2.0:1775764728.028729:0:61577:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 4/2,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051b31 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.028743:0:61577:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 4/2,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051b31 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1775764728.028754:0:61577:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764728.028757:0:61577:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 7/2,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051b31 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.028764:0:61577:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 6/1,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051b31 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1775764728.028770:0:61577:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764728.028777:0:61577:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x6:0x0] updating i_size 14 i_blocks 11 00000020:00010000:2.0:1775764728.028781:0:61577:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c920 (ffff8b23812ec6c8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764728.028784:0:61577:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c920 (ffff8b23812ec6c8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764728.028787:0:61577:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24430 (ffff8b23812eccf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764728.028791:0:61577:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c920 (ffff8b23812ec6c8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764728.028793:0:61577:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24430 (ffff8b23812ec6c8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764728.028950:0:61577:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:0 00000008:00010000:2.0:1775764728.029002:0:61577:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b2383c9ee40 00000008:00010000:2.0:1775764728.029065:0:46654:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b2383c9ee40 00010000:00010000:2.0:1775764728.029089:0:46654:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051b31 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00100000:2.0:1775764728.029130:0:46654:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea5f80 - object 0x280000400:281 offset 0<>4096 00000100:00100000:2.0:1775764728.029210:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764728.029227:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098596992:192.168.202.128@tcp:3:lfs.0 00000008:00010000:2.0:1775764728.032656:0:2378:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8b2383c9ee40 00000020:00200000:2.0:1775764728.032672:0:2378:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1775764728.032693:0:2378:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8b2383c9ee40 00000100:00100000:2.0:1775764728.032724:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098596992:192.168.202.128@tcp:3:lfs.0 00000100:00100000:2.0:1775764728.037188:0:61577:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61577:1862022098597248:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764728.037224:0:61577:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b2382dac380 x1862022098589824/t34359739120(34359739120) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764742 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764728.191588:0:61588:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764728.191619:0:61588:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764728.191687:0:61588:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 61588 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764728.191699:0:61588:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61588 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764728.191710:0:61588:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 61588 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764728.191738:0:61588:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61588:1862022098597376:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764728.191955:0:61588:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384904f00 going to sleep for 11 seconds 00000100:00100000:1.0:1775764728.198145:0:61588:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61588:1862022098597376:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764728.198174:0:61588:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764728.198188:0:61588:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764728.198205:0:61588:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764728.198215:0:61588:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764728.198228:0:61588:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764728.198235:0:61588:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764728.198242:0:61588:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764728.198254:0:61588:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864ff480 x1862022098597376/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764744 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764728.198267:0:61588:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764728.198280:0:61588:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1775764728.198287:0:61588:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764728.198314:0:61588:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:1.0:1775764728.198322:0:61588:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bcff 00000080:00010000:1.0:1775764728.198329:0:61588:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764728.198333:0:61588:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bcff from it ffffa60b0202fbb0 00010000:00010000:1.0:1775764728.198336:0:61588:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764728.198344:0:61588:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764728.198371:0:61588:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764728.198384:0:61588:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:1.0:1775764728.198437:0:61588:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffd3cc5ca50 00000100:00100000:1.0:1775764728.198550:0:61588:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61588:1862022098597504:192.168.202.128@tcp:34:lfs.0 00000100:00100000:1.0:1775764728.198643:0:61588:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384904f00 going to sleep for 11 seconds 00000100:00100000:1.0:1775764728.214019:0:61588:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff480 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61588:1862022098597504:192.168.202.128@tcp:34:lfs.0 00000080:00200008:1.0:1775764728.214097:0:61588:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200008:2.0:1775764728.405192:0:61601:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764728.405207:0:61601:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764728.405229:0:61601:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764728.405248:0:61601:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764728.405269:0:61601:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764728.405277:0:61601:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764728.405285:0:61601:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764728.405289:0:61601:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bcff 00000080:00010000:2.0:1775764728.405296:0:61601:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bcff from it ffffa60b0203fcb8 00010000:00010000:2.0:1775764728.405300:0:61601:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764728.405309:0:61601:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764728.405334:0:61601:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764728.405340:0:61601:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764728.405346:0:61601:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764728.405354:0:61601:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764728.405360:0:61601:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764728.405364:0:61601:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764728.405366:0:61601:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bcff 00000080:00010000:2.0:1775764728.405369:0:61601:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bcff from it ffffa60b0203fb58 00010000:00010000:2.0:1775764728.405372:0:61601:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764728.405379:0:61601:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764728.405386:0:61601:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764728.405392:0:61601:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f22.sanity-pcc, flags=8 00000080:00200008:2.0:1775764728.405397:0:61601:0:(file.c:6226:ll_getattr_dentry()) START file f22.sanity-pcc:[0x200003ab4:0x6:0x0](ffff8b2389a61148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764728.405400:0:61601:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148),name=f22.sanity-pcc 00000002:00010000:2.0:1775764728.405411:0:61601:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764728.405418:0:61601:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bd06 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051b70 expref: -99 pid: 61595 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764728.405426:0:61601:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bd06 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051b70 expref: -99 pid: 61595 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764728.405432:0:61601:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764728.405436:0:61601:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) 00000080:00010000:2.0:1775764728.405438:0:61601:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) for lock 0x73cfd86cfc17bd06 00000080:00010000:2.0:1775764728.405442:0:61601:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd06 from it ffffa60b0203fcb8 00010000:00010000:2.0:1775764728.405445:0:61601:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bd06 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051b70 expref: -99 pid: 61595 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764728.405488:0:61601:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bd06 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051b70 expref: -99 pid: 61595 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764728.405520:0:61601:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764728.405526:0:61601:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764728.405534:0:61601:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764728.405542:0:61601:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764728.405548:0:61601:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764728.405551:0:61601:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764728.405553:0:61601:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bcff 00000080:00010000:2.0:1775764728.405557:0:61601:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bcff from it ffffa60b0203fbf8 00010000:00010000:2.0:1775764728.405559:0:61601:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764728.405576:0:61601:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764728.405585:0:61601:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764728.405589:0:61601:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f22.sanity-pcc, flags=265 00000080:00200000:2.0:1775764728.405593:0:61601:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148), inode mode 81a4 mask 46 00000080:00200008:2.0:1775764728.405598:0:61601:0:(file.c:1057:ll_file_open()) START file f22.sanity-pcc:[0x200003ab4:0x6:0x0](ffff8b2389a61148), flags 154402 00000002:00010000:2.0:1775764728.405612:0:61601:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x6:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240114403 00010000:00010000:2.0:1775764728.405668:0:61601:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea800/0x73cfd86cfc17bd0d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 61601 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.405675:0:61601:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea800/0x73cfd86cfc17bd0d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61601 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764728.405687:0:61601:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764728.405700:0:61601:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5f80 x1862022098598272/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764728.405710:0:61601:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea800/0x73cfd86cfc17bd0d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 61601 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764728.405734:0:61601:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61601:1862022098598272:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764728.405964:0:61601:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391386180 going to sleep for 11 seconds 00000100:00100000:1.0:1775764728.420630:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119049344 00000100:00100000:1.0:1775764728.420639:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119049344 00000100:00100000:1.0:1775764728.420696:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 244 00000100:00100000:1.0:1775764728.420705:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dac700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119049344:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764728.420723:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17bd06 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03051b70 expref: -99 pid: 61595 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764728.420798:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764728.420805:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dac700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119049344:12345-192.168.202.128@tcp:104: Request processed in 101us (358us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764728.420811:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 244 00000100:00100000:2.0:1775764728.427710:0:61601:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:61601:1862022098598272:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764728.427740:0:61601:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CW ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea800/0x73cfd86cfc17bd0d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b85 expref: -99 pid: 61601 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.427757:0:61601:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea800/0x73cfd86cfc17bd0d lrc: 4/1,0 mode: --/CW res: [0x200003ab4:0x6:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b85 expref: -99 pid: 61601 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.427765:0:61601:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea800/0x73cfd86cfc17bd0d lrc: 4/1,0 mode: --/CW res: [0x200003ab4:0x6:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b85 expref: -99 pid: 61601 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.427777:0:61601:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea800/0x73cfd86cfc17bd0d lrc: 4/1,0 mode: CW/CW res: [0x200003ab4:0x6:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b85 expref: -99 pid: 61601 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.427787:0:61601:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea800/0x73cfd86cfc17bd0d lrc: 4/1,0 mode: CW/CW res: [0x200003ab4:0x6:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b85 expref: -99 pid: 61601 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.427800:0:61601:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea800/0x73cfd86cfc17bd0d lrc: 5/1,1 mode: CW/CW res: [0x200003ab4:0x6:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b85 expref: -99 pid: 61601 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.427808:0:61601:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea800/0x73cfd86cfc17bd0d lrc: 5/1,1 mode: CW/CW res: [0x200003ab4:0x6:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b85 expref: -99 pid: 61601 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.427819:0:61601:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea800/0x73cfd86cfc17bd0d lrc: 4/0,1 mode: CW/CW res: [0x200003ab4:0x6:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b85 expref: -99 pid: 61601 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764728.427829:0:61601:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23b9ea5f80 x1862022098598272/t34359739140(34359739140) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764744 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764728.427841:0:61601:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea5f80 x1862022098598272/t34359739140(34359739140) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764744 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764728.427855:0:61601:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea800/0x73cfd86cfc17bd0d lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0x6:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b85 expref: -99 pid: 61601 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.427865:0:61601:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 8 res 8589949620/6 (0 0) 00000080:00200000:2.0:1775764728.427872:0:61601:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764728.427983:0:61601:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x6:0x0](ffff8b2389a61148) for lock 0x73cfd86cfc17bd0d 00000080:00010000:2.0:1775764728.427991:0:61601:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd0d from it ffffa60b0203fbf0 00010000:00010000:2.0:1775764728.427995:0:61601:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea800/0x73cfd86cfc17bd0d lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0x6:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b85 expref: -99 pid: 61601 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764728.428003:0:61601:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea800/0x73cfd86cfc17bd0d lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x6:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b85 expref: -99 pid: 61601 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:2.0:1775764728.428030:0:61601:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea5f80 x1862022098598272/t34359739140(34359739140) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764744 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764728.428118:0:61601:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x6:0x0](ffff8b2389a61148) cmd=c00466fc arg=7ffffdacd06c 00000080:00200008:2.0:1775764728.428183:0:61601:0:(file.c:447:ll_file_release()) START file f22.sanity-pcc:[0x200003ab4:0x6:0x0](ffff8b2389a61148), flags 154002 00010000:00010000:2.0:1775764728.428203:0:61601:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea800/0x73cfd86cfc17bd0d lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x6:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b85 expref: -99 pid: 61601 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764729.785857:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764729.925126:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2378:1862022098598528:192.168.202.128@tcp:400:kworker.0 00010000:00010000:1.0:1775764730.810240:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764732.857386:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764733.163007:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764733.163097:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764733.165125:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764733.165132:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b238a3bf000 lock: 00000000da82aaac/0x73cfd86cfc17bc50 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x57b8ff0e03051916 expref: -99 pid: 61542 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764733.165140:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8740000 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:2.0:1775764733.165146:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b238a3bf000 lock: 00000000da82aaac/0x73cfd86cfc17bc50 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x428400010000 nid: local remote: 0x57b8ff0e03051916 expref: -99 pid: 61542 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764733.165232:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b238a3bf000 lock: 00000000da82aaac/0x73cfd86cfc17bc50 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x57b8ff0e03051916 expref: -99 pid: 61542 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764733.165239:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764733.165262:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b238a3bf000 lock: 00000000da82aaac/0x73cfd86cfc17bc50 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x57b8ff0e03051916 expref: -99 pid: 61542 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764733.165270:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b238a3bf000 lock: 00000000da82aaac/0x73cfd86cfc17bc50 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x118:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x57b8ff0e03051916 expref: -99 pid: 61542 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764733.165291:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764733.165292:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764733.165294:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764733.888675:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1775764734.064749:0:62016:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1775764734.064762:0:62016:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764734.064782:0:62016:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764734.064801:0:62016:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764734.064817:0:62016:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764734.064825:0:62016:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764734.064836:0:62016:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764734.064840:0:62016:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bcff 00000080:00010000:2.0:1775764734.064847:0:62016:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bcff from it ffffa60b01c0bcb8 00010000:00010000:2.0:1775764734.064852:0:62016:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764734.064861:0:62016:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200008:2.0:1775764734.075126:0:62016:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1775764734.075137:0:62016:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:2.0:1775764734.075162:0:62016:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764734.075230:0:62016:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 62016 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764734.075241:0:62016:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62016 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764734.075252:0:62016:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62016 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764734.075279:0:62016:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lctl:82781e18-e0e3-4f9f-9a2b-041bf0344414:62016:1862022098599552:192.168.202.128@tcp:101:lctl.0 00000100:00100000:2.0:1775764734.079251:0:62016:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23ac48a480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764734.096678:0:62016:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lctl:82781e18-e0e3-4f9f-9a2b-041bf0344414:62016:1862022098599552:192.168.202.128@tcp:101:lctl.0 00010000:00010000:2.0:1775764734.096737:0:62016:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764734.096762:0:62016:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764734.096774:0:62016:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764734.096797:0:62016:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764734.096809:0:62016:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764734.096832:0:62016:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764734.096838:0:62016:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764734.096862:0:62016:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea5880 x1862022098599552/t0(0) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764750 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lctl.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764734.096895:0:62016:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764734.096959:0:62016:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2387c07800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764734.096966:0:62016:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764734.097062:0:62016:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764734.097087:0:62016:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17bd14 00000080:00010000:2.0:1775764734.097110:0:62016:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:2.0:1775764734.097113:0:62016:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd14 from it ffffa60b01c0bcb8 00010000:00010000:2.0:1775764734.097116:0:62016:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764734.097125:0:62016:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764734.097425:0:62016:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764734.097443:0:62016:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764734.097493:0:62016:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764734.097506:0:62016:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764734.097523:0:62016:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764734.097529:0:62016:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764734.097531:0:62016:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bcff 00000080:00010000:2.0:1775764734.097535:0:62016:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bcff from it ffffa60b01c0bbb0 00010000:00010000:2.0:1775764734.097537:0:62016:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764734.097549:0:62016:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764734.097558:0:62016:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764734.097567:0:62016:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200080:2.0:1775764734.097589:0:62016:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=8080667f arg=7ffcddfba650 00000080:00200008:2.0:1775764734.097603:0:62016:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200000:2.0:1775764734.098289:0:62016:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764734.098301:0:62016:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764734.098309:0:62016:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764734.098320:0:62016:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764734.098326:0:62016:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764734.098330:0:62016:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764734.098333:0:62016:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bcff 00000080:00010000:2.0:1775764734.098337:0:62016:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bcff from it ffffa60b01c0bbb0 00010000:00010000:2.0:1775764734.098339:0:62016:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764734.098346:0:62016:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1775764734.098373:0:62016:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764734.098380:0:62016:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200080:2.0:1775764734.098406:0:62016:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=8080667f arg=7ffcddfb9530 00000080:00200008:2.0:1775764734.098414:0:62016:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200000:2.0:1775764734.098722:0:62016:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764734.098732:0:62016:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764734.098752:0:62016:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764734.098771:0:62016:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1775764734.098778:0:62016:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764734.098783:0:62016:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764734.098786:0:62016:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bcff 00000080:00010000:2.0:1775764734.098789:0:62016:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bcff from it ffffa60b01c0bbb0 00010000:00010000:2.0:1775764734.098792:0:62016:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764734.098800:0:62016:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1775764734.098808:0:62016:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764734.098837:0:62016:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 100000 00000080:00200080:2.0:1775764734.099180:0:62016:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c01466fc arg=7ffcddfb7410 00800000:00000080:2.0:1775764734.099221:0:62016:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d9 len=56 karg=ffff8b238872e400 uarg=0000000000000000 00000002:00000080:2.0:1775764734.099244:0:62016:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d9 len=56 karg=ffff8b238872e400 uarg=0000000000000000 00010000:00010000:2.0:1775764734.099275:0:62016:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764734.099382:0:62016:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x6:0x0]: losing layout lock 00010000:00010000:2.0:1775764734.099544:0:62016:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764734.099575:0:62016:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:2.0:1775764734.099613:0:62016:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job lctl:fcf15866-4739-4595-8fe1-44abde7ee9d7:62016:1862022098599808:192.168.202.128@tcp:103:lctl.0 00000100:00100000:2.0:1775764734.099866:0:62016:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23ac48a480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764734.130166:0:62016:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job lctl:fcf15866-4739-4595-8fe1-44abde7ee9d7:62016:1862022098599808:192.168.202.128@tcp:103:lctl.0 00010000:00010000:2.0:1775764734.130201:0:62016:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bcf1 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051b2a expref: -99 pid: 61577 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000020:00100000:2.0:1775764734.130232:0:62016:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 58, max 7 00000100:00100000:2.0:1775764734.130241:0:62016:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5880 x1862022098599936/t0(0) o58->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764734.130525:0:62016:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lctl:fcf15866-4739-4595-8fe1-44abde7ee9d7:62016:1862022098599936:192.168.202.128@tcp:58:lctl.0 00000100:00100000:1.0:1775764734.130755:0:62016:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2390abc540 going to sleep for 11 seconds 00000100:00100000:2.0:1775764734.141365:0:62016:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 34359739140 gen 1 00000100:00100000:2.0:1775764734.141632:0:62016:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23b9ea5f80 x1862022098598272/t34359739140(34359739140) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764744 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764734.141658:0:62016:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lctl:fcf15866-4739-4595-8fe1-44abde7ee9d7:62016:1862022098599936:192.168.202.128@tcp:58:lctl.0 00000080:00200080:1.0:1775764734.231737:0:62023:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=400866d7 arg=7fa5a0000b78 00800000:00000080:1.0:1775764734.231755:0:62023:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b01c0bcf0 uarg=0000000000000000 00000002:00000080:1.0:1775764734.231770:0:62023:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b01c0bcf0 uarg=0000000000000000 00000020:00100000:1.0:1775764734.231808:0:62023:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 57, max 7 00000100:00100000:1.0:1775764734.231827:0:62023:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fea00 x1862022098600960/t0(0) o57->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764734.231858:0:62023:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:62023:1862022098600960:192.168.202.128@tcp:57:lhsmtool_posix.0 00000100:00100000:1.0:1775764734.232280:0:62023:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2390abca80 going to sleep for 11 seconds 00000100:00100000:1.0:1775764734.258759:0:62023:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:62023:1862022098600960:192.168.202.128@tcp:57:lhsmtool_posix.0 00000080:00200008:1.0:1775764734.258818:0:62023:0:(file.c:447:ll_file_release()) START file 0x200003ab4:0x6:0x0:[0x200003ab4:0x6:0x0](ffff8b2389a61148), flags 1504000 00000002:00100000:1.0:1775764734.258879:0:62023:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b2382daea00 x1862022098600704/t34359739141(34359739141) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764750 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764734.258889:0:62023:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764734.258895:0:62023:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fea00 x1862022098601216/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764734.258917:0:62023:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:62023:1862022098601216:192.168.202.128@tcp:35:lhsmtool_posix.0 00000100:00100000:1.0:1775764734.259097:0:62023:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2390abca80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764734.319004:0:61408:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61408:1862022098601344:192.168.202.128@tcp:60:lhsmtool_posix.0 00000100:00100000:2.0:1775764734.319123:0:61408:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23ac48a480 going to sleep for 11 seconds 00000100:00100000:1.0:1775764734.329312:0:61408:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61408:1862022098601344:192.168.202.128@tcp:60:lhsmtool_posix.0 00000080:00200008:1.0:1775764734.329388:0:61408:0:(file.c:447:ll_file_release()) START file fid:[0x200000002:0x2:0x0](ffff8b2390ec5348), flags 100000 00000002:00100000:1.0:1775764734.329440:0:61408:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864f0700 x1862022098557568/t34359739034(34359739034) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/4688 e 0 to 0 dl 1775764737 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764734.329451:0:61408:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764734.329472:0:61408:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fe300 x1862022098601600/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764734.329492:0:61408:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:61408:1862022098601600:192.168.202.128@tcp:35:lhsmtool_posix.0 00000100:00100000:1.0:1775764734.329573:0:61408:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2390abc9c0 going to sleep for 11 seconds 00000080:00200008:2.0:1775764734.622089:0:62042:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764734.622101:0:62042:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:2.0:1775764734.622120:0:62042:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764734.622138:0:62042:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764734.622153:0:62042:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764734.622162:0:62042:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764734.622171:0:62042:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:2.0:1775764734.622175:0:62042:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17bd14 00000080:00010000:2.0:1775764734.622182:0:62042:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd14 from it ffffa60b01d93cb8 00010000:00010000:2.0:1775764734.622185:0:62042:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764734.622195:0:62042:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200008:1.0:1775764734.622799:0:62042:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764734.622806:0:62042:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764734.622820:0:62042:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764734.622831:0:62042:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764734.622842:0:62042:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764734.622849:0:62042:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764734.622855:0:62042:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764734.622858:0:62042:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bcff 00000080:00010000:1.0:1775764734.622862:0:62042:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bcff from it ffffa60b01d93cb8 00010000:00010000:1.0:1775764734.622865:0:62042:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764734.622873:0:62042:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1775764734.903932:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764735.035309:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2378:1862022098601856:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764735.035428:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098602112:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764735.035488:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098602368:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764735.035540:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:2.0:1775764735.035547:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2378:1862022098601728:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764735.035575:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2378:1862022098601984:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764735.035597:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098602240:192.168.202.128@tcp:400:kworker.0 00010000:00010000:2.0:1775764736.952588:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764738.999586:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1775764739.818050:0:62201:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-pcc test 23: Test write/truncate/mmap-write invalidating RO-PCC caching ========================================================== 15:58:57 (1775764737) 00010000:00010000:1.0:1775764741.055231:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1775764741.819436:0:62303:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764741.819538:0:62303:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764741.819638:0:62303:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764741.819657:0:62303:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1775764741.819666:0:62303:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764741.819675:0:62303:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764741.819678:0:62303:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bcff 00000080:00010000:1.0:1775764741.819684:0:62303:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bcff from it ffffa60b01dcbbb0 00010000:00010000:1.0:1775764741.819688:0:62303:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764741.819697:0:62303:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764741.819711:0:62303:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764741.819718:0:62303:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 100000 00000080:00200000:1.0:1775764741.819748:0:62303:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764741.819755:0:62303:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764741.819761:0:62303:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764741.819769:0:62303:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764741.819774:0:62303:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764741.819779:0:62303:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764741.819781:0:62303:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bcff 00000080:00010000:1.0:1775764741.819784:0:62303:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bcff from it ffffa60b01dcbbf8 00010000:00010000:1.0:1775764741.819787:0:62303:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764741.819793:0:62303:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1775764741.819801:0:62303:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764741.819807:0:62303:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=.lustre, flags=17 00000080:00200000:1.0:1775764741.819813:0:62303:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000002:0x1:0x0](ffff8b2389909988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764741.819824:0:62303:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=fid, dir=[0x200000002:0x1:0x0](ffff8b2389909988), file ffff8b23a0fbaa80, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1775764741.819829:0:62303:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=fid, dir=[0x200000002:0x1:0x0](ffff8b2389909988), intent=open 00000002:00010000:1.0:1775764741.819842:0:62303:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: fid,[0x0:0x0:0x0]) in obj [0x200000002:0x1:0x0], intent: open flags 0100001 00010000:00010000:1.0:1775764741.819891:0:62303:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd29 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 62303 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764741.819898:0:62303:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd29 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62303 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764741.819909:0:62303:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764741.819921:0:62303:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fca80 x1862022098603392/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764741.819930:0:62303:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd29 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62303 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764741.819951:0:62303:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:62303:1862022098603392:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764741.822318:0:62303:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23894b6480 going to sleep for 11 seconds 00000100:00100000:1.0:1775764741.860592:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098603648:192.168.202.128@tcp:103:ldlm_bl.0 00000080:00200000:2.0:1775764741.967764:0:62311:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764741.967788:0:62311:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764741.967803:0:62311:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1775764741.967816:0:62311:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:2.0:1775764741.967823:0:62311:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764741.967829:0:62311:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764741.967832:0:62311:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bcff 00000080:00010000:2.0:1775764741.967838:0:62311:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bcff from it ffffa60b01c13bb0 00010000:00010000:2.0:1775764741.967840:0:62311:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1775764741.967846:0:62311:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1775764741.967856:0:62311:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764741.967862:0:62311:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200080:2.0:1775764741.967881:0:62311:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=8080667f arg=7fffa6f54d60 00000080:00200008:2.0:1775764741.967892:0:62311:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200000:2.0:1775764741.986997:0:62071:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764741.987023:0:62071:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764741.987042:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1775764741.987057:0:62071:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1775764741.987066:0:62071:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764741.987074:0:62071:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764741.987077:0:62071:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bcff 00000080:00010000:2.0:1775764741.987085:0:62071:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bcff from it ffffa60b01d7bbf8 00010000:00010000:2.0:1775764741.987089:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1775764741.987097:0:62071:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1775764741.987111:0:62071:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764741.987127:0:62071:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764741.987133:0:62071:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764741.987139:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1775764741.987147:0:62071:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1775764741.987154:0:62071:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764741.987158:0:62071:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764741.987159:0:62071:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bcff 00000080:00010000:2.0:1775764741.987163:0:62071:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bcff from it ffffa60b01d7bb38 00010000:00010000:2.0:1775764741.987165:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1775764741.987169:0:62071:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:2.0:1775764741.987174:0:62071:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:2.0:1775764741.987178:0:62071:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b238b731500, open_flags 8241, mode 81b6 opened 0 00000080:00200000:2.0:1775764741.987183:0:62071:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000002:00010000:2.0:1775764741.987199:0:62071:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f23.sanity-pcc,[0x200003ab4:0x9:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0101102 00010000:00010000:2.0:1775764741.987208:0:62071:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:2.0:1775764741.987259:0:62071:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:2.0:1775764741.987264:0:62071:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764741.987266:0:62071:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17bcff lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051b3f expref: -99 pid: 61588 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:2.0:1775764741.987292:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17bd30 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 62071 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764741.987296:0:62071:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17bd30 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62071 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764741.987305:0:62071:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764741.987317:0:62071:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7800 x1862022098603904/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764741.987328:0:62071:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17bd30 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62071 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764741.987352:0:62071:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:62071:1862022098603904:192.168.202.128@tcp:101:bash.0 00000100:00100000:2.0:1775764741.987546:0:62071:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1db40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764741.993839:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764741.994367:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764741.994374:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764741.994383:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6eb200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764741.994388:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764741.994397:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764741.994406:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764741.994496:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 2/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764741.994503:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764741.994535:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764741.994542:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd14 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03051ba1 expref: -99 pid: 62016 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764741.994564:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764741.994566:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764741.994567:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764741.994575:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764741.994589:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2378:1862022098604032:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764742.015078:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2378:1862022098604032:192.168.202.128@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764742.072014:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764742.118631:0:62314:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62314:1862022098604544:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764742.118659:0:62314:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.118671:0:62314:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.118684:0:62314:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.118696:0:62314:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.118709:0:62314:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.118717:0:62314:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.118728:0:62314:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764742.118740:0:62314:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b2382daf480 x1862022098604544/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764758 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764742.118755:0:62314:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.118767:0:62314:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764742.118774:0:62314:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764742.118805:0:62314:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764742.118811:0:62314:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bd3e 00000080:00010000:2.0:1775764742.118816:0:62314:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764742.118819:0:62314:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd3e from it ffffa60b01e33b98 00010000:00010000:2.0:1775764742.118822:0:62314:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.118831:0:62314:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764742.118855:0:62314:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764742.118867:0:62314:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f23.sanity-pcc, flags=8 00000080:00200000:2.0:1775764742.208752:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764742.208769:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764742.208826:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764742.208854:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x3:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764742.208860:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764742.208880:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x3:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764742.208898:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764742.208902:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764742.208917:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764742.208933:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764742.208939:0:8:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x6:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764742.208942:0:8:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764742.208944:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764742.208959:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764742.208976:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x8:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764742.208980:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764742.208997:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x8:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764742.209015:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764742.209019:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:2.0:1775764742.209041:0:8:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 00000000cf960b96 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1775764742.209060:0:8:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000cf960b96 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764742.209111:0:8:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00100000:2.0:1775764742.209190:0:8:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea6a00 - object 0x240000400:282 offset 0<>11 00000100:00100000:2.0:1775764742.209256:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098604672:192.168.202.128@tcp:4:bash.0 00000100:00100000:2.0:1775764742.209391:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764742.209400:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098604800:192.168.202.128@tcp:16:kworker.0 00000100:00100000:2.0:1775764742.219507:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098604800:192.168.202.128@tcp:16:kworker.0 00000008:00010000:2.0:1775764742.237875:0:2378:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 00000000cf960b96 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:2.0:1775764742.237921:0:2378:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000cf960b96 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1775764742.237957:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098604672:192.168.202.128@tcp:4:bash.0 00010000:00010000:2.0:1775764742.413052:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764742.413060:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17bd45 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051c2d expref: -99 pid: 62330 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.413066:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5fe00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764742.413071:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17bd45 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051c2d expref: -99 pid: 62330 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.413079:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17bd45 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051c2d expref: -99 pid: 62330 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.413086:0:46149:0:(ldlm_request.c:1765:ldlm_cli_cancel()) ### lock is already being destroyed 00010000:00010000:2.0:1775764742.413089:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17bd45 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051c2d expref: -99 pid: 62330 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.413095:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17bd45 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051c2d expref: -99 pid: 62330 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.413112:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764742.413113:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764742.413115:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764742.421646:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098605440:192.168.202.128@tcp:3:lfs.0 00010000:00010000:2.0:1775764742.966620:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764742.966627:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bd5a lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03051c8f expref: -99 pid: 62361 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.966634:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23c2044a00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764742.966638:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bd5a lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03051c8f expref: -99 pid: 62361 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.966647:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bd5a lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03051c8f expref: -99 pid: 62361 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.966656:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bd5a lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03051c8f expref: -99 pid: 62361 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.966673:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e800/0x73cfd86cfc17bd53 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051c7a expref: -99 pid: 62354 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764742.966707:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bd5a lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051c8f expref: -99 pid: 62361 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.966714:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764742.966751:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bd5a lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051c8f expref: -99 pid: 62361 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.966764:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044a00/0x73cfd86cfc17bd5a lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051c8f expref: -99 pid: 62361 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764742.966778:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764742.966779:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764742.966781:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764742.966789:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764742.966805:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098606720:192.168.202.128@tcp:103:ldlm_bl.0 00000080:00200008:2.0:1775764742.967817:0:62361:0:(file.c:447:ll_file_release()) START file f23.sanity-pcc:[0x200003ab4:0x9:0x0](ffff8b23899c1988), flags 100000 00000002:00100000:2.0:1775764742.967864:0:62361:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b2382dad180 x1862022098606592/t34359739153(34359739153) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764758 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'mmap_cat.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764742.967875:0:62361:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764742.967880:0:62361:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4a80 x1862022098606976/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764742.967899:0:62361:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job mmap_cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:62361:1862022098606976:192.168.202.128@tcp:35:mmap_cat.0 00000100:00100000:2.0:1775764742.967944:0:62361:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1dcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764742.972240:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098606720:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764742.972941:0:62361:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job mmap_cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:62361:1862022098606976:192.168.202.128@tcp:35:mmap_cat.0 00000002:00100000:2.0:1775764742.972979:0:62361:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b2382dad180 x1862022098606592/t34359739153(34359739153) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764758 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'mmap_cat.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764743.095571:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1775764743.103067:0:62371:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764743.103097:0:62371:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764743.103116:0:62371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764743.103132:0:62371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764743.103141:0:62371:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764743.103149:0:62371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764743.103153:0:62371:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bd3e 00000080:00010000:1.0:1775764743.103159:0:62371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd3e from it ffffa60b01ed3bb0 00010000:00010000:1.0:1775764743.103163:0:62371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764743.103171:0:62371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764743.103189:0:62371:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764743.103197:0:62371:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:1.0:1775764743.103227:0:62371:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffdd0a7b680 00000080:00200008:1.0:1775764743.103259:0:62371:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:2.0:1775764743.149196:0:62071:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764743.149223:0:62071:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764743.149242:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764743.149257:0:62071:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1775764743.149265:0:62071:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764743.149272:0:62071:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764743.149276:0:62071:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bd3e 00000080:00010000:2.0:1775764743.149283:0:62071:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd3e from it ffffa60b01d7bbf8 00010000:00010000:2.0:1775764743.149286:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764743.149295:0:62071:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1775764743.149329:0:62071:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764743.149341:0:62071:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f23.sanity-pcc, flags=777 00000080:00200000:2.0:1775764743.149348:0:62071:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x9:0x0](ffff8b23899c1988), inode mode 81a4 mask 42 00000080:00200008:2.0:1775764743.149356:0:62071:0:(file.c:1057:ll_file_open()) START file f23.sanity-pcc:[0x200003ab4:0x9:0x0](ffff8b23899c1988), flags 101101 00000002:00010000:2.0:1775764743.149371:0:62071:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x9:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 04240101102 00010000:00010000:2.0:1775764743.149393:0:62071:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1775764743.149853:0:62071:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1775764743.149861:0:62071:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764743.149864:0:62071:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bd3e lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051c1f expref: -99 pid: 62314 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1775764743.149892:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 62071 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.149900:0:62071:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62071 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764743.149911:0:62071:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764743.149923:0:62071:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4a80 x1862022098607232/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764743.149932:0:62071:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62071 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764743.149954:0:62071:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:62071:1862022098607232:192.168.202.128@tcp:101:bash.0 00000100:00100000:2.0:1775764743.150098:0:62071:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1dcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764743.159786:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764743.159835:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119052928 00000100:00100000:1.0:1775764743.159845:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119052928 00000100:00100000:1.0:1775764743.159897:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 248 00000100:00100000:1.0:1775764743.159907:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23864fe300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42325:x1862022119052928:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764743.159923:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e800/0x73cfd86cfc17bd53 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03051c7a expref: -99 pid: 62354 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764743.160011:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764743.160018:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23864fe300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42325:x1862022119052928:12345-192.168.202.128@tcp:104: Request processed in 113us (255us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764743.160028:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 248 00010000:00010000:2.0:1775764743.160280:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764743.160287:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e800/0x73cfd86cfc17bd53 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03051c7a expref: -99 pid: 62354 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764743.160306:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5e800 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764743.160312:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e800/0x73cfd86cfc17bd53 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03051c7a expref: -99 pid: 62354 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764743.160320:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e800/0x73cfd86cfc17bd53 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03051c7a expref: -99 pid: 62354 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764743.160329:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e800/0x73cfd86cfc17bd53 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03051c7a expref: -99 pid: 62354 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764743.160356:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bd4c lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051c49 expref: -99 pid: 62330 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764743.160396:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e800/0x73cfd86cfc17bd53 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03051c7a expref: -99 pid: 62354 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764743.160403:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764743.160437:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e800/0x73cfd86cfc17bd53 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03051c7a expref: -99 pid: 62354 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764743.160444:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e800/0x73cfd86cfc17bd53 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03051c7a expref: -99 pid: 62354 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764743.160481:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764743.160483:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764743.160485:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764743.168521:0:62071:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:62071:1862022098607232:192.168.202.128@tcp:101:bash.0 00010000:00010000:2.0:1775764743.168562:0:62071:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 4/0,1 mode: --/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ca4 expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.168574:0:62071:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 4/0,1 mode: --/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ca4 expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.168587:0:62071:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 4/0,1 mode: CW/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ca4 expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.168598:0:62071:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 4/0,1 mode: CW/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ca4 expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764743.168615:0:62071:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=200002b, status=0 req@ffff8b23b9ea4a80 x1862022098607232/t34359739155(34359739155) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764759 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764743.168627:0:62071:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4a80 x1862022098607232/t34359739155(34359739155) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764759 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764743.168643:0:62071:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ca4 expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.168654:0:62071:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 8 res 8589949620/9 (0 0) 00000080:00200000:2.0:1775764743.168661:0:62071:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764743.168755:0:62071:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x9:0x0](ffff8b23899c1988) for lock 0x73cfd86cfc17bd61 00000080:00010000:2.0:1775764743.168762:0:62071:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd61 from it ffffa60b01d7bbf0 00010000:00010000:2.0:1775764743.168766:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ca4 expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.168775:0:62071:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ca4 expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:2.0:1775764743.168801:0:62071:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4a80 x1862022098607232/t34359739155(34359739155) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764759 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1775764743.168860:0:62071:0:(llite_lib.c:2274:ll_setattr_raw()) START file f23.sanity-pcc:[0x200003ab4:0x9:0x0](ffff8b23899c1988) current size 11, valid attrs a068, mode 0, uid 2, gid 0, new size 0, atime 1775764743.000000000, mtime 1775764743.000000000, ctime 1775764743.000000000 00010000:00010000:2.0:1775764743.168885:0:62071:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bd4c lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03051c49 expref: -99 pid: 62330 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764743.168912:0:62071:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x9:0x0]: losing layout lock 00010000:00010000:2.0:1775764743.168986:0:62071:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bd4c lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051c49 expref: -99 pid: 62330 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764743.168994:0:62071:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764743.168996:0:62071:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bd4c lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051c49 expref: -99 pid: 62330 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000020:00100000:2.0:1775764743.169015:0:62071:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764743.169022:0:62071:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4000 x1862022098607616/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764743.169042:0:62071:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:62071:1862022098607616:192.168.202.128@tcp:36:bash.0 00000100:00100000:2.0:1775764743.169132:0:62071:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1dcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764743.185315:0:62071:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:62071:1862022098607616:192.168.202.128@tcp:36:bash.0 00010000:00010000:2.0:1775764743.185363:0:62071:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 31 res 8589949620/9 (0 0) 00000080:00010000:2.0:1775764743.185374:0:62071:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x9:0x0](ffff8b23899c1988) 00000002:00010000:2.0:1775764743.185392:0:62071:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x9:0x0]) in obj [0x200003ab4:0x9:0x0], intent: layout flags 0 00010000:00010000:2.0:1775764743.185450:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17bd68 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 62071 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.185493:0:62071:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17bd68 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62071 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.185504:0:62071:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17bd68 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62071 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764743.185525:0:62071:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:62071:1862022098607744:192.168.202.128@tcp:101:bash.0 00000002:00010000:2.0:1775764743.215535:0:62071:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bd6f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cc7 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764743.215558:0:62071:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bd6f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cc7 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.215571:0:62071:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/9 (0 0) 00000080:00200000:2.0:1775764743.215578:0:62071:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x9:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764743.215586:0:62071:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x9:0x0](ffff8b23899c1988) for lock 0x73cfd86cfc17bd6f 00000080:00010000:2.0:1775764743.215593:0:62071:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd6f from it ffffa60b01d7b930 00010000:00010000:2.0:1775764743.215597:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bd6f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051cc7 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.215606:0:62071:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bd6f lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cc7 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764743.215629:0:62071:0:(file.c:1815:ll_io_set_mirror()) f23.sanity-pcc: desiginated mirror: 0 00000080:00200000:2.0:1775764743.215636:0:62071:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 2 restore needed 0 00010000:00010000:2.0:1775764743.215650:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bd6f lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cc7 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764743.215664:0:62071:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bd6f lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cc7 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764743.215672:0:62071:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x9:0x0](000000007ac2eb40) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bd6f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cc7 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764743.215765:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bd6f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051cc7 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764743.215778:0:62071:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bd6f lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cc7 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764743.215806:0:62071:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764743.215814:0:62071:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764743.215817:0:62071:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764743.215822:0:62071:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764743.215870:0:62071:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764743.215895:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bd6f lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cc7 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764743.215904:0:62071:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bd6f lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cc7 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764743.215910:0:62071:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x9:0x0](000000007ac2eb40) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bd6f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cc7 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764743.215943:0:62071:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764743.215948:0:62071:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764743.215950:0:62071:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764743.215952:0:62071:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764743.215979:0:62071:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:2.0:1775764743.215999:0:62071:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 11 blocks 1 [cma]time: 1775764742 1775764742 0 by oinfo size 11 blocks 1 [cma]time 1775764742 1775764742 1775764742 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000080:00200000:2.0:1775764743.216062:0:62071:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x9:0x0]: layout version change: 4294967294 -> 3 00010000:00010000:2.0:1775764743.216075:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bd6f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051cc7 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764743.216083:0:62071:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bd6f lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051cc7 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00020000:00200000:2.0:1775764743.216095:0:62071:0:(lov_io.c:385:lov_io_mirror_init()) [0x200003ab4:0x9:0x0] need write intent for [0, 18446744073709551615) 00000080:00200000:2.0:1775764743.216099:0:62071:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 3 need write layout 1, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764743.216102:0:62071:0:(vvp_io.c:355:vvp_io_fini()) [0x200003ab4:0x9:0x0] write layout, type 3 [0x0, 0x1) 00000080:00010000:2.0:1775764743.216109:0:62071:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x9:0x0](ffff8b23899c1988) 00000002:00010000:2.0:1775764743.216117:0:62071:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x9:0x0]) in obj [0x200003ab4:0x9:0x0], intent: layout flags 02 00010000:00010000:2.0:1775764743.216132:0:62071:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bd6f lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03051cc7 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764743.216155:0:62071:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x9:0x0]: losing layout lock 00010000:00010000:2.0:1775764743.216193:0:62071:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bd6f lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051cc7 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764743.216200:0:62071:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764743.216203:0:62071:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bd6f lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051cc7 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764743.216223:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x9:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 62071 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.216227:0:62071:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62071 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764743.216237:0:62071:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764743.216245:0:62071:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4380 x1862022098608256/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/880 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764743.216255:0:62071:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62071 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764743.216277:0:62071:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:62071:1862022098608256:192.168.202.128@tcp:101:bash.0 00000100:00100000:2.0:1775764743.216386:0:62071:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1d240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764743.313332:0:62071:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:62071:1862022098608256:192.168.202.128@tcp:101:bash.0 00010000:00010000:2.0:1775764743.313359:0:62071:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 4/0,1 mode: --/EX res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.313372:0:62071:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 4/0,1 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.313385:0:62071:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.313396:0:62071:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.313411:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.313419:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.313439:0:62071:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764743.313500:0:62071:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea4380 x1862022098608256/t34359739159(34359739159) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/600 e 0 to 0 dl 1775764759 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764743.313520:0:62071:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764743.313532:0:62071:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.313544:0:62071:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/9 (0 0) 00000080:00200000:2.0:1775764743.313550:0:62071:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x9:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764743.313560:0:62071:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x9:0x0](ffff8b23899c1988) for lock 0x73cfd86cfc17bd76 00000080:00010000:2.0:1775764743.313566:0:62071:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd76 from it ffffa60b01d7b930 00010000:00010000:2.0:1775764743.313570:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.313578:0:62071:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764743.313595:0:62071:0:(file.c:1815:ll_io_set_mirror()) f23.sanity-pcc: desiginated mirror: 0 00000080:00200000:2.0:1775764743.313602:0:62071:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 3 restore needed 0 00010000:00010000:2.0:1775764743.313613:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764743.313623:0:62071:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764743.313636:0:62071:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x9:0x0](000000007ac2eb40) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764743.313715:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764743.313722:0:62071:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764743.313746:0:62071:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764743.313753:0:62071:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764743.313756:0:62071:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764743.313761:0:62071:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764743.313812:0:62071:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764743.313833:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764743.313842:0:62071:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764743.313849:0:62071:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x9:0x0](000000007ac2eb40) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764743.313877:0:62071:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764743.313880:0:62071:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764743.313882:0:62071:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764743.313885:0:62071:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764743.313908:0:62071:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764743.314277:0:62071:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x9:0x0]: layout version change: 4294967294 -> 4 00010000:00010000:2.0:1775764743.314288:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764743.314296:0:62071:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00020000:00200000:2.0:1775764743.314308:0:62071:0:(lov_io.c:308:lov_io_mirror_write_intent()) [0x200003ab4:0x9:0x0]there are 0 components to be staled to modify file extent [0x0, 0xffffffffffffffff), iot: 3 00020000:00200000:2.0:1775764743.314312:0:62071:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764743.314314:0:62071:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764743.314319:0:62071:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764743.314335:0:62071:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:2.0:1775764743.314346:0:62071:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 4 00000008:00010000:2.0:1775764743.314350:0:62071:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24e30, osc lock ffff8b238a3f9c80, flags 0x80000000 00010000:00010000:2.0:1775764743.314364:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1775764743.314374:0:62071:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1775764743.314384:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000008:00010000:2.0:1775764743.314394:0:62071:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 11, setting rss=11; kms=11, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1775764743.314409:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1775764743.314416:0:62071:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000080:00200000:2.0:1775764743.330008:0:62071:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x9:0x0] updating i_size 0 i_blocks 1 00000020:00010000:2.0:1775764743.330017:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384cd6218 (ffff8b238a3fff38/0) at cl_lock_release():245 00000020:00010000:2.0:1775764743.330020:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384cd6218 (ffff8b238a3fff38/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764743.330025:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24e30 (ffff8b23812ecbd8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764743.330035:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1775764743.330050:0:62071:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000020:00010000:2.0:1775764743.330063:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384cd6218 (ffff8b238a3fff38/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764743.330066:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24e30 (ffff8b238a3fff38/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764743.330112:0:62071:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1775764743.330137:0:62071:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file f23.sanity-pcc:[0x200003ab4:0x9:0x0](ffff8b23899c1988) current size 0, valid attrs a168, mode 0, uid 2, gid 0, new size 0, atime 1775764743.000000000, mtime 1775764743.000000000, ctime 1775764743.000000000, rc 0 00000080:00200008:2.0:1775764743.330221:0:62071:0:(file.c:2612:do_file_write_iter()) START file f23.sanity-pcc:[0x200003ab4:0x9:0x0], ppos: 0, count: 14 00000080:00200000:2.0:1775764743.330270:0:62071:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 14 bytes 00000080:00200000:2.0:1775764743.330277:0:62071:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 14 00000080:00200000:2.0:1775764743.330288:0:62071:0:(file.c:2058:ll_file_io_generic()) f23.sanity-pcc: write ppos: 0, bytes: 14 00000080:00200000:2.0:1775764743.330294:0:62071:0:(file.c:1815:ll_io_set_mirror()) f23.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764743.330301:0:62071:0:(cl_io.c:214:cl_io_rw_init()) header@000000002e91068a[0x110, 1, [0x200003ab4:0x9:0x0] hash] 00000020:00200000:2.0:1775764743.330303:0:62071:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 14) 0 0 00000080:00200000:2.0:1775764743.330306:0:62071:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764743.330331:0:62071:0:(lov_io.c:308:lov_io_mirror_write_intent()) [0x200003ab4:0x9:0x0]there are 0 components to be staled to modify file extent [0x0, 0xffffffffffffffff), iot: 2 00020000:00200000:2.0:1775764743.330336:0:62071:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764743.330339:0:62071:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:2.0:1775764743.330344:0:62071:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:2.0:1775764743.330350:0:62071:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 14) 14, 14 00020000:00200000:2.0:1775764743.330354:0:62071:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764743.330380:0:62071:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1775764743.330395:0:62071:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764743.330400:0:62071:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24e30, osc lock ffff8b238a3f9c80, flags 0x0 00010000:00010000:2.0:1775764743.330423:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1775764743.330486:0:62071:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1775764743.330502:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1775764743.330510:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1775764743.330517:0:62071:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000080:00200000:2.0:1775764743.330529:0:62071:0:(vvp_io.c:1286:vvp_io_write_start()) f23.sanity-pcc: write [0, 14) append 0 append lockpos 0 00000080:00200000:2.0:1775764743.330539:0:62071:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 14 bytes 00000080:00200000:2.0:1775764743.330588:0:62071:0:(rw26.c:858:ll_write_end()) pos 0, len 14, copied 14 00000080:00200000:2.0:1775764743.330596:0:62071:0:(rw26.c:916:ll_write_end()) page@00000000d1ce3a9a[3 000000007b9b600d 2 1 0000000024d4832d] 00000080:00200000:2.0:1775764743.330599:0:62071:0:(rw26.c:916:ll_write_end()) vmpage @0000000023418de7 17ffffc0001001 3:0 ffff8b23baeaec40 0 lru 00000080:00200000:2.0:1775764743.330606:0:62071:0:(rw26.c:916:ll_write_end()) osc-page@00000000c66b2839 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 000000003596ffcd 00000000336aac03 > 3< 0 0 > 4< 0 0 8 8519680 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764743.330608:0:62071:0:(rw26.c:916:ll_write_end()) end page@00000000d1ce3a9a 00000080:00200000:2.0:1775764743.330609:0:62071:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764743.330613:0:62071:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 14 prio 0 00000008:00010000:2.0:1775764743.330630:0:62071:0:(osc_cache.c:848:osc_extent_find()) ### extent: 000000008a7bf9d4 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000080:00200000:2.0:1775764743.330647:0:62071:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b23899c1b38, count 1, dirtied 1 00000080:00200000:2.0:1775764743.330650:0:62071:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 14 bytes, tot: 14 00000080:00200000:2.0:1775764743.330653:0:62071:0:(vvp_io.c:1353:vvp_io_write_start()) f23.sanity-pcc: write bytes 0, result: 14 00000020:00010000:2.0:1775764743.330668:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36bc68 (ffff8b23812ecbd8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764743.330670:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36bc68 (ffff8b23812ecbd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764743.330672:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24e30 (ffff8b23812ec368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764743.330676:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1775764743.330683:0:62071:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00010000:2.0:1775764743.330691:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36bc68 (ffff8b23812ecbd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764743.330694:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24e30 (ffff8b23812ecbd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764743.330699:0:62071:0:(vvp_io.c:517:vvp_io_advance()) advancing 14 bytes 00000080:00200000:2.0:1775764743.330703:0:62071:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:2.0:1775764743.330717:0:62071:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764743.330720:0:62071:0:(file.c:2206:ll_file_io_generic()) f23.sanity-pcc: 2 io complete with rc: 0, result: 14, restart: 0 00000080:00200000:2.0:1775764743.330726:0:62071:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x9:0x0] updating i_size 14 i_blocks 1 00000080:00200000:2.0:1775764743.330730:0:62071:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 14 00000080:00200008:2.0:1775764743.330756:0:62071:0:(file.c:447:ll_file_release()) START file f23.sanity-pcc:[0x200003ab4:0x9:0x0](ffff8b23899c1988), flags 100001 00010000:00010000:2.0:1775764743.330781:0:62071:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ca4 expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764743.349732:0:62372:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764743.349759:0:62372:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764743.349823:0:62372:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 62372 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764743.349834:0:62372:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62372 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764743.349844:0:62372:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62372 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764743.349869:0:62372:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62372:1862022098608512:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764743.350083:0:62372:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23894b69c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764743.359137:0:62372:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 34359739159 gen 1 00000100:00100000:2.0:1775764743.359154:0:62372:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23864fca80 x1862022098603392/t34359739144(34359739144) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/4688 e 0 to 0 dl 1775764757 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764743.359186:0:62372:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23b9ea4a80 x1862022098607232/t34359739155(34359739155) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764759 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764743.359205:0:62372:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62372:1862022098608512:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764743.359222:0:62372:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.359234:0:62372:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.359247:0:62372:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.359257:0:62372:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.359269:0:62372:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.359277:0:62372:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.359285:0:62372:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764743.359294:0:62372:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864fea00 x1862022098608512/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764759 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764743.359306:0:62372:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.359328:0:62372:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764743.359333:0:62372:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764743.359358:0:62372:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764743.359365:0:62372:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bd7d 00000080:00010000:2.0:1775764743.359371:0:62372:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764743.359373:0:62372:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd7d from it ffffa60b01eb3bf8 00010000:00010000:2.0:1775764743.359376:0:62372:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.359385:0:62372:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764743.359407:0:62372:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764743.359441:0:62372:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b2383aab200, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1775764743.359449:0:62372:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open 00000002:00010000:2.0:1775764743.359503:0:62372:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f23.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:2.0:1775764743.359559:0:62372:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd84 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 62372 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.359566:0:62372:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd84 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62372 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764743.359576:0:62372:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764743.359582:0:62372:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4380 x1862022098608768/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764743.359589:0:62372:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd84 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62372 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764743.359608:0:62372:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62372:1862022098608768:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764743.360149:0:62372:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1d240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764743.364419:0:62372:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62372:1862022098608768:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764743.364493:0:62372:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd84 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62372 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.364507:0:62372:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd84 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 62372 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.364514:0:62372:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd84 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 62372 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.364520:0:62372:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd84 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 62372 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.364528:0:62372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd84 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 62372 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.364535:0:62372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6eb600 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764743.364540:0:62372:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd84 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 62372 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.364546:0:62372:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd84 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 62372 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.364552:0:62372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd84 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 62372 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.364559:0:62372:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764743.364564:0:62372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd84 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 62372 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.364571:0:62372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd84 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 62372 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764743.364592:0:62372:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8b23b9ea4380 x1862022098608768/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/608 e 0 to 0 dl 1775764759 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764743.364606:0:62372:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764743.364746:0:62372:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764743.364767:0:62372:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764743.364781:0:62372:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764743.364793:0:62372:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764743.364801:0:62372:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764743.364806:0:62372:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764743.364810:0:62372:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bd7d 00000080:00010000:2.0:1775764743.364814:0:62372:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd7d from it ffffa60b01eb3bb0 00010000:00010000:2.0:1775764743.364818:0:62372:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764743.364825:0:62372:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764743.364839:0:62372:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764743.364850:0:62372:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764743.364882:0:62372:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086915 arg=114a3d0 00000080:00200000:1.0:1775764743.645878:0:62389:0:(file.c:2058:ll_file_io_generic()) f23.sanity-pcc: read ppos: 14, bytes: 4194290 00000080:00200000:1.0:1775764743.645885:0:62389:0:(file.c:1815:ll_io_set_mirror()) f23.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764743.645892:0:62389:0:(cl_io.c:214:cl_io_rw_init()) header@000000002e91068a[0x110, 2, [0x200003ab4:0x9:0x0] hash] 00000020:00200000:1.0:1775764743.645894:0:62389:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [14, 4194304) 0 0 00000080:00200000:1.0:1775764743.645897:0:62389:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764743.645906:0:62389:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764743.645909:0:62389:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:1.0:1775764743.645915:0:62389:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 14 chunk: [14, 4194304) 4194304, 4194290 00020000:00200000:1.0:1775764743.645919:0:62389:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764743.645946:0:62389:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 1023] 00000008:00010000:1.0:1775764743.645956:0:62389:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764743.645961:0:62389:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5bb0, osc lock ffff8b2382cf6130, flags 0x400000000000000 00010000:00010000:1.0:1775764743.645975:0:62389:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764743.645986:0:62389:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4194303) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764743.645996:0:62389:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764743.646003:0:62389:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764743.646010:0:62389:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000080:00200000:1.0:1775764743.646019:0:62389:0:(vvp_io.c:826:vvp_io_read_start()) f23.sanity-pcc: read [14, 4194304) 00000001:00010000:1.0:1775764743.646030:0:62389:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x9:0x0] 00000008:00010000:1.0:1775764743.646035:0:62389:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764743.646038:0:62389:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5330, osc lock ffff8b2382cf6e40, flags 0x400000400001002 00010000:00010000:1.0:1775764743.646044:0:62389:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764743.646053:0:62389:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000020:00200000:1.0:1775764743.646062:0:62389:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764743.646065:0:62389:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 8/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764743.646073:0:62389:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 7/1,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000020:00200000:1.0:1775764743.646080:0:62389:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764743.646083:0:62389:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x9:0x0] updating i_size 14 i_blocks 1 00000020:00010000:1.0:1775764743.646086:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f19918 (ffff8b2383fb5cf8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764743.646088:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f19918 (ffff8b2383fb5cf8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764743.646091:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5330 (ffff8b2387cba368/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764743.646093:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f19918 (ffff8b2383fb5cf8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764743.646095:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5330 (ffff8b2383fb5cf8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764743.646103:0:62389:0:(vvp_io.c:885:vvp_io_read_start()) count:4194290 iocb pos:14 00000020:00010000:1.0:1775764743.646113:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb7b78 (ffff8b2383fb5cf8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764743.646116:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb7b78 (ffff8b2383fb5cf8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764743.646118:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5bb0 (ffff8b2387cba368/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764743.646122:0:62389:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764743.646130:0:62389:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00000020:00010000:1.0:1775764743.646141:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb7b78 (ffff8b2383fb5cf8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764743.646143:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5bb0 (ffff8b2383fb5cf8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764743.646149:0:62389:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:1.0:1775764743.646172:0:62389:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764743.646175:0:62389:0:(file.c:2206:ll_file_io_generic()) f23.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:1.0:1775764743.646178:0:62389:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:1.0:1775764743.646300:0:62389:0:(file.c:2454:do_file_read_iter()) START file f23.sanity-pcc:[0x200003ab4:0x9:0x0], ppos: 14, count: 4194304 00000080:00200000:1.0:1775764743.646327:0:62389:0:(file.c:2058:ll_file_io_generic()) f23.sanity-pcc: read ppos: 14, bytes: 4194304 00000080:00200000:1.0:1775764743.646331:0:62389:0:(file.c:1815:ll_io_set_mirror()) f23.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764743.646335:0:62389:0:(cl_io.c:214:cl_io_rw_init()) header@000000002e91068a[0x110, 2, [0x200003ab4:0x9:0x0] hash] 00000020:00200000:1.0:1775764743.646337:0:62389:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [14, 4194318) 0 0 00000080:00200000:1.0:1775764743.646339:0:62389:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764743.646345:0:62389:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764743.646347:0:62389:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:1.0:1775764743.646350:0:62389:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 14 chunk: [14, 4194318) 4194318, 4194304 00020000:00200000:1.0:1775764743.646353:0:62389:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764743.646366:0:62389:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 1024] 00000008:00010000:1.0:1775764743.646388:0:62389:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764743.646391:0:62389:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5bb0, osc lock ffff8b2382cf6130, flags 0x400000000000000 00010000:00010000:1.0:1775764743.646400:0:62389:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764743.646409:0:62389:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4198399) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764743.646417:0:62389:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764743.646432:0:62389:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764743.646439:0:62389:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00000080:00200000:1.0:1775764743.646447:0:62389:0:(vvp_io.c:826:vvp_io_read_start()) f23.sanity-pcc: read [14, 4194318) 00000001:00010000:1.0:1775764743.646489:0:62389:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x9:0x0] 00000008:00010000:1.0:1775764743.646494:0:62389:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764743.646496:0:62389:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5330, osc lock ffff8b2382cf6e40, flags 0x400000400001002 00010000:00010000:1.0:1775764743.646503:0:62389:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764743.646511:0:62389:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00000020:00200000:1.0:1775764743.646519:0:62389:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764743.646522:0:62389:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 8/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764743.646528:0:62389:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 7/1,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00000020:00200000:1.0:1775764743.646534:0:62389:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764743.646539:0:62389:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x9:0x0] updating i_size 14 i_blocks 1 00000020:00010000:1.0:1775764743.646541:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_release():245 00000020:00010000:1.0:1775764743.646543:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764743.646546:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5330 (ffff8b2387cba008/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764743.646548:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764743.646550:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5330 (ffff8b2387cba368/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764743.646555:0:62389:0:(vvp_io.c:885:vvp_io_read_start()) count:4194304 iocb pos:14 00000020:00010000:1.0:1775764743.646560:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb7b78 (ffff8b2387cba368/0) at cl_lock_release():245 00000020:00010000:1.0:1775764743.646563:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb7b78 (ffff8b2387cba368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764743.646565:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5bb0 (ffff8b2387cba008/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764743.646568:0:62389:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764743.646576:0:62389:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00000020:00010000:1.0:1775764743.646584:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb7b78 (ffff8b2387cba368/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764743.646586:0:62389:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5bb0 (ffff8b2387cba368/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764743.646591:0:62389:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:1.0:1775764743.646603:0:62389:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764743.646607:0:62389:0:(file.c:2206:ll_file_io_generic()) f23.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:1.0:1775764743.646609:0:62389:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:1.0:1775764743.646682:0:62389:0:(file.c:447:ll_file_release()) START file f23.sanity-pcc:[0x200003ab4:0x9:0x0](ffff8b23899c1988), flags 100000 00000002:00100000:1.0:1775764743.646735:0:62389:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b2382dad500 x1862022098609280/t34359739160(34359739160) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764759 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cat.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764743.646747:0:62389:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764743.646752:0:62389:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fed80 x1862022098609536/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764743.646791:0:62389:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:62389:1862022098609536:192.168.202.128@tcp:35:cat.0 00000100:00100000:1.0:1775764743.646896:0:62389:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23894b6c00 going to sleep for 11 seconds 00000080:00200008:2.0:1775764743.759919:0:62395:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764743.759933:0:62395:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764743.759959:0:62395:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764743.759977:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764743.759992:0:62395:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764743.759999:0:62395:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764743.760007:0:62395:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764743.760010:0:62395:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bd7d 00000080:00010000:2.0:1775764743.760018:0:62395:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd7d from it ffffa60b01f13cb8 00010000:00010000:2.0:1775764743.760021:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764743.760030:0:62395:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764743.760061:0:62395:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764743.760068:0:62395:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764743.760074:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764743.760082:0:62395:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764743.760088:0:62395:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764743.760092:0:62395:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764743.760094:0:62395:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bd7d 00000080:00010000:2.0:1775764743.760110:0:62395:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd7d from it ffffa60b01f13b58 00010000:00010000:2.0:1775764743.760113:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764743.760120:0:62395:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764743.760128:0:62395:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764743.760140:0:62395:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), flags=8 00000080:00200000:2.0:1775764743.760144:0:62395:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=getattr 00000002:00010000:2.0:1775764743.760156:0:62395:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f23.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764743.760198:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 62395 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.760206:0:62395:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62395 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.760219:0:62395:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62395 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764743.760247:0:62395:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62395:1862022098609664:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764743.760354:0:62395:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1d240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764743.778689:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764743.779246:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764743.779254:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03051ca4 expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764743.779262:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6eb200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764743.779267:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03051ca4 expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764743.779274:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03051ca4 expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764743.779282:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 3/0,0 mode: CW/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x57b8ff0e03051ca4 expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000002:00100000:2.0:1775764743.779345:0:46149:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea4a80 x1862022098607232/t34359739155(34359739155) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764759 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764743.779356:0:46149:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764743.779361:0:46149:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7800 x1862022098609920/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764743.779384:0:46149:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:46149:1862022098609920:192.168.202.128@tcp:35:ldlm_bl.0 00000100:00100000:2.0:1775764743.779508:0:46149:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1d9c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764743.790583:0:46149:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:46149:1862022098609920:192.168.202.128@tcp:35:ldlm_bl.0 00000002:00100000:2.0:1775764743.790619:0:46149:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea4a80 x1862022098607232/t34359739155(34359739155) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764759 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764743.790719:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 2/0,0 mode: --/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03051ca4 expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764743.790729:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764743.790762:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 1/0,0 mode: --/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03051ca4 expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764743.790779:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd61 lrc: 0/0,0 mode: --/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03051ca4 expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764743.790799:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764743.790801:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764743.790803:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764743.803268:0:62395:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62395:1862022098609664:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764743.803295:0:62395:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d61 expref: -99 pid: 62395 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.803322:0:62395:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d61 expref: -99 pid: 62395 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.803330:0:62395:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d61 expref: -99 pid: 62395 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.803344:0:62395:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051d61 expref: -99 pid: 62395 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.803356:0:62395:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d61 expref: -99 pid: 62395 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.803369:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051d61 expref: -99 pid: 62395 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.803377:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051d61 expref: -99 pid: 62395 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.803383:0:62395:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051d61 expref: -99 pid: 62395 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764743.803394:0:62395:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea4000 x1862022098609664/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/840 e 0 to 0 dl 1775764759 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764743.803406:0:62395:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d61 expref: -99 pid: 62395 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764743.803430:0:62395:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d61 expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.803443:0:62395:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/9 (0 0) 00000080:00200000:2.0:1775764743.803451:0:62395:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764743.803599:0:62395:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0x9:0x0](ffff8b23899c1988): rc = 0 00000080:00200000:2.0:1775764743.803621:0:62395:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x9:0x0]: layout version change: 4 -> 4 00000080:00010000:2.0:1775764743.803631:0:62395:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x9:0x0](ffff8b23899c1988) for lock 0x73cfd86cfc17bd92 00000080:00010000:2.0:1775764743.803644:0:62395:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x9:0x0](ffff8b23899c1988) 00000080:00010000:2.0:1775764743.803648:0:62395:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd92 from it ffffa60b01f13b20 00010000:00010000:2.0:1775764743.803652:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051d61 expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.803660:0:62395:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051d61 expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1775764743.803696:0:62395:0:(file.c:6226:ll_getattr_dentry()) START file f23.sanity-pcc:[0x200003ab4:0x9:0x0](ffff8b23899c1988), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764743.803702:0:62395:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x9:0x0](ffff8b23899c1988),name=f23.sanity-pcc 00000002:00010000:2.0:1775764743.803722:0:62395:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x9:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764743.803731:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bd8b lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051d3e expref: -99 pid: 62389 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764743.803741:0:62395:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bd8b lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051d3e expref: -99 pid: 62389 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764743.803748:0:62395:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764743.803753:0:62395:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x9:0x0](ffff8b23899c1988) 00000080:00010000:2.0:1775764743.803756:0:62395:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x9:0x0](ffff8b23899c1988) for lock 0x73cfd86cfc17bd8b 00000080:00010000:2.0:1775764743.803759:0:62395:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd8b from it ffffa60b01f13cb8 00010000:00010000:2.0:1775764743.803762:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bd8b lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051d3e expref: -99 pid: 62389 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764743.803768:0:62395:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bd8b lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051d3e expref: -99 pid: 62389 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764743.803806:0:62395:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:2.0:1775764743.803817:0:62395:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764743.803820:0:62395:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1775764743.803824:0:62395:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x9:0x0] 00000008:00010000:2.0:1775764743.803850:0:62395:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764743.803853:0:62395:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24e30, osc lock ffff8b238a3f9c80, flags 0x400000400001002 00010000:00010000:2.0:1775764743.803883:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1775764743.803895:0:62395:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00000020:00200000:2.0:1775764743.803907:0:62395:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764743.803910:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1775764743.803918:0:62395:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00000020:00200000:2.0:1775764743.803926:0:62395:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764743.803937:0:62395:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x9:0x0] updating i_size 14 i_blocks 1 00000020:00010000:2.0:1775764743.803941:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027e910 (ffff8b23812ecbd8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764743.803944:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027e910 (ffff8b23812ecbd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764743.803947:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24e30 (ffff8b23812ec368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764743.803951:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027e910 (ffff8b23812ecbd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764743.803954:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24e30 (ffff8b23812ecbd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764743.803974:0:62395:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/1, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764743.804027:0:62395:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764743.804035:0:62395:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764743.804042:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764743.804051:0:62395:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764743.804057:0:62395:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764743.804061:0:62395:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764743.804063:0:62395:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bd7d 00000080:00010000:2.0:1775764743.804067:0:62395:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd7d from it ffffa60b01f13bf8 00010000:00010000:2.0:1775764743.804070:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764743.804077:0:62395:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764743.804086:0:62395:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764743.804092:0:62395:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f23.sanity-pcc, flags=265 00000080:00200000:2.0:1775764743.804097:0:62395:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x9:0x0](ffff8b23899c1988), inode mode 81a4 mask 44 00000080:00200008:2.0:1775764743.804104:0:62395:0:(file.c:1057:ll_file_open()) START file f23.sanity-pcc:[0x200003ab4:0x9:0x0](ffff8b23899c1988), flags 100000 00000002:00010000:2.0:1775764743.804118:0:62395:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x9:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 04240100001 00010000:00010000:2.0:1775764743.804161:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd99 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 62395 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.804168:0:62395:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd99 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62395 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764743.804179:0:62395:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764743.804187:0:62395:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4000 x1862022098610304/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764743.804201:0:62395:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd99 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62395 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764743.804220:0:62395:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62395:1862022098610304:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764743.804393:0:62395:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1d240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764743.829757:0:62395:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62395:1862022098610304:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764743.829813:0:62395:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd99 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d6f expref: -99 pid: 62395 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.829825:0:62395:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd99 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d6f expref: -99 pid: 62395 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.829838:0:62395:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd99 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051d6f expref: -99 pid: 62395 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.829851:0:62395:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd99 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d6f expref: -99 pid: 62395 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764743.829878:0:62395:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23b9ea4000 x1862022098610304/t34359739163(34359739163) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764759 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764743.829896:0:62395:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4000 x1862022098610304/t34359739163(34359739163) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764759 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764743.829911:0:62395:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd99 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d6f expref: -99 pid: 62395 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764743.829929:0:62395:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd99 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d6f expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.829941:0:62395:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/9 (0 0) 00000080:00200000:2.0:1775764743.829949:0:62395:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764743.830073:0:62395:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x9:0x0]: layout version change: 4 -> 4 00000080:00010000:2.0:1775764743.830086:0:62395:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x9:0x0](ffff8b23899c1988) for lock 0x73cfd86cfc17bd99 00000080:00010000:2.0:1775764743.830093:0:62395:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd99 from it ffffa60b01f13bf0 00010000:00010000:2.0:1775764743.830097:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd99 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051d6f expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.830107:0:62395:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd99 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051d6f expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00100000:2.0:1775764743.830130:0:62395:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4000 x1862022098610304/t34359739163(34359739163) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764759 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764743.830232:0:62395:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x9:0x0](ffff8b23899c1988) cmd=400866fc arg=7ffdea855ad8 00000080:00010000:2.0:1775764743.830769:0:62395:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x9:0x0](ffff8b23899c1988) 00000002:00010000:2.0:1775764743.830780:0:62395:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x9:0x0]) in obj [0x200003ab4:0x9:0x0], intent: layout flags 02 00010000:00010000:2.0:1775764743.830798:0:62395:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03051d61 expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764743.830823:0:62395:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x9:0x0]: losing layout lock 00010000:00010000:2.0:1775764743.830844:0:62395:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bd8b lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1a/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03051d3e expref: -99 pid: 62389 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764743.830923:0:62395:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x9:0x0]: losing layout lock 00010000:00010000:2.0:1775764743.830940:0:62395:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd99 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xd/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03051d6f expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764743.830964:0:62395:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x9:0x0]: losing layout lock 00010000:00010000:2.0:1775764743.830976:0:62395:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764743.830991:0:62395:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x9:0x0]: losing layout lock 00010000:00010000:2.0:1775764743.831026:0:62395:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051d61 expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764743.831034:0:62395:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bd8b lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051d3e expref: -99 pid: 62389 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764743.831040:0:62395:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd99 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051d6f expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764743.831046:0:62395:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764743.831051:0:62395:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 4 locks packed 00010000:00010000:2.0:1775764743.831054:0:62395:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17bd92 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051d61 expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764743.831069:0:62395:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045a00/0x73cfd86cfc17bd8b lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051d3e expref: -99 pid: 62389 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764743.831079:0:62395:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bd99 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051d6f expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764743.831088:0:62395:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bd76 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051cd5 expref: -99 pid: 62071 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764743.831117:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 62395 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.831125:0:62395:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62395 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764743.831136:0:62395:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764743.831144:0:62395:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4a80 x1862022098610560/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 400/880 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764743.831154:0:62395:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62395 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764743.831177:0:62395:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62395:1862022098610560:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764743.831806:0:62395:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1d240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764743.838303:0:62395:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62395:1862022098610560:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764743.838342:0:62395:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 4/0,1 mode: --/EX res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.838356:0:62395:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 4/0,1 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.838381:0:62395:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.838394:0:62395:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.838416:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.838434:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.838442:0:62395:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764743.838488:0:62395:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea4a80 x1862022098610560/t34359739164(34359739164) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 400/600 e 0 to 0 dl 1775764759 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764743.838510:0:62395:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764743.838522:0:62395:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.838535:0:62395:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/9 (0 0) 00000080:00200000:2.0:1775764743.838540:0:62395:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x9:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764743.838549:0:62395:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x9:0x0](ffff8b23899c1988) for lock 0x73cfd86cfc17bda0 00000080:00010000:2.0:1775764743.838557:0:62395:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bda0 from it ffffa60b01f13c10 00010000:00010000:2.0:1775764743.838561:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764743.838570:0:62395:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764743.838598:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764743.838608:0:62395:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764743.838615:0:62395:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x9:0x0](000000007ac2eb40) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764743.838704:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764743.838711:0:62395:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764743.838738:0:62395:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764743.838747:0:62395:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764743.838749:0:62395:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764743.838754:0:62395:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:2.0:1775764743.838788:0:62395:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 000000008a7bf9d4 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00000008:00010000:2.0:1775764743.838801:0:62395:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 000000008a7bf9d4 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1775764743.838844:0:62395:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00000008:00100000:2.0:1775764743.839825:0:62395:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea7b80 - object 0x240000400:282 offset 0<>14 00000080:00200000:2.0:1775764743.849752:0:62395:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764743.849875:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764743.849891:0:62395:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764743.849903:0:62395:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x9:0x0](000000007ac2eb40) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764743.849973:0:62395:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764743.849983:0:62395:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764743.849986:0:62395:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764743.849991:0:62395:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764743.850038:0:62395:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:2.0:1775764743.850058:0:62395:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 11 blocks 1 [cma]time: 1775764742 1775764742 1775764742 by oinfo size 14 blocks 1 [cma]time 1775764743 1775764743 1775764743 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00000080:00200000:2.0:1775764743.850125:0:62395:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x9:0x0]: layout version change: 4294967294 -> 5 00010000:00010000:2.0:1775764743.850136:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764743.850146:0:62395:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bda0 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051d8b expref: -99 pid: 62395 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:2.0:1775764743.850603:0:62395:0:(file.c:2454:do_file_read_iter()) START file f23.sanity-pcc:[0x200003ab4:0x9:0x0], ppos: 0, count: 1048576 00000080:00200000:2.0:1775764743.850632:0:62395:0:(rw.c:1928:ll_readpage()) fast read pgno: 0 00000080:00200000:2.0:1775764743.850652:0:62395:0:(file.c:2058:ll_file_io_generic()) f23.sanity-pcc: read ppos: 0, bytes: 1048576 00000080:00200000:2.0:1775764743.850657:0:62395:0:(file.c:1815:ll_io_set_mirror()) f23.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764743.850664:0:62395:0:(cl_io.c:214:cl_io_rw_init()) header@000000002e91068a[0x110, 1, [0x200003ab4:0x9:0x0] hash] 00000020:00200000:2.0:1775764743.850666:0:62395:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 1048576) 0 0 00000080:00200000:2.0:1775764743.850669:0:62395:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764743.850678:0:62395:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 10, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764743.850682:0:62395:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764743.850688:0:62395:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:2.0:1775764743.850691:0:62395:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764743.850710:0:62395:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:2.0:1775764743.850722:0:62395:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764743.850728:0:62395:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24830, osc lock ffff8b238a3f9c80, flags 0x400000000000000 00010000:00010000:2.0:1775764743.850744:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764743.850755:0:62395:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764743.850765:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00000008:00010000:2.0:1775764743.850774:0:62395:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 14, setting rss=14; kms=14, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764743.850784:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764743.850791:0:62395:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00000080:00200000:2.0:1775764743.850800:0:62395:0:(vvp_io.c:826:vvp_io_read_start()) f23.sanity-pcc: read [0, 1048576) 00000001:00010000:2.0:1775764743.850814:0:62395:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x9:0x0] 00000008:00010000:2.0:1775764743.850819:0:62395:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764743.850821:0:62395:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24db0, osc lock ffff8b238a3f9300, flags 0x400000400001002 00010000:00010000:2.0:1775764743.850829:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764743.850837:0:62395:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00000020:00200000:2.0:1775764743.850845:0:62395:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764743.850849:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764743.850856:0:62395:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00000020:00200000:2.0:1775764743.850862:0:62395:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764743.850869:0:62395:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x9:0x0] updating i_size 14 i_blocks 1 00000020:00010000:2.0:1775764743.850873:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027e910 (ffff8b23812ecbd8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764743.850877:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027e910 (ffff8b23812ecbd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764743.850879:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24db0 (ffff8b23812ec368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764743.850883:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027e910 (ffff8b23812ecbd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764743.850886:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24db0 (ffff8b23812ecbd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764743.850893:0:62395:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:0 00000008:00010000:2.0:1775764743.850932:0:62395:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b2383c9e300 00000008:00010000:2.0:1775764743.850988:0:46654:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b2383c9e300 00010000:00010000:2.0:1775764743.851008:0:46654:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00000008:00100000:2.0:1775764743.851050:0:46654:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea7100 - object 0x240000400:282 offset 0<>4096 00000020:00010000:2.0:1775764743.856088:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36bc68 (ffff8b23812ecbd8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764743.856093:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36bc68 (ffff8b23812ecbd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764743.856098:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24830 (ffff8b23812ec368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764743.856110:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764743.856128:0:62395:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00000020:00010000:2.0:1775764743.856143:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36bc68 (ffff8b23812ecbd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764743.856146:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24830 (ffff8b23812ecbd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764743.856156:0:62395:0:(vvp_io.c:517:vvp_io_advance()) advancing 14 bytes 00000080:00200000:2.0:1775764743.856186:0:62395:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 5 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764743.856191:0:62395:0:(file.c:2206:ll_file_io_generic()) f23.sanity-pcc: 1 io complete with rc: 0, result: 14, restart: 0 00000080:00200000:2.0:1775764743.856197:0:62395:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 14 00000080:00200008:2.0:1775764743.858037:0:62395:0:(file.c:2454:do_file_read_iter()) START file f23.sanity-pcc:[0x200003ab4:0x9:0x0], ppos: 14, count: 1048576 00000080:00200000:2.0:1775764743.858082:0:62395:0:(file.c:2058:ll_file_io_generic()) f23.sanity-pcc: read ppos: 14, bytes: 1048576 00000080:00200000:2.0:1775764743.858088:0:62395:0:(file.c:1815:ll_io_set_mirror()) f23.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764743.858096:0:62395:0:(cl_io.c:214:cl_io_rw_init()) header@000000002e91068a[0x110, 2, [0x200003ab4:0x9:0x0] hash] 00000020:00200000:2.0:1775764743.858100:0:62395:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [14, 1048590) 0 0 00000080:00200000:2.0:1775764743.858104:0:62395:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764743.858120:0:62395:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 10, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764743.858124:0:62395:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764743.858132:0:62395:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 14 chunk: [14, 1048590) 1048590, 1048576 00020000:00200000:2.0:1775764743.858137:0:62395:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764743.858168:0:62395:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 256] 00000008:00010000:2.0:1775764743.858188:0:62395:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764743.858194:0:62395:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24830, osc lock ffff8b238a3f9c80, flags 0x400000000000000 00010000:00010000:2.0:1775764743.858224:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764743.858255:0:62395:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1052671) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764743.858271:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764743.858281:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764743.858290:0:62395:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00000080:00200000:2.0:1775764743.858299:0:62395:0:(vvp_io.c:826:vvp_io_read_start()) f23.sanity-pcc: read [14, 1048590) 00000001:00010000:2.0:1775764743.858326:0:62395:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x9:0x0] 00000008:00010000:2.0:1775764743.858332:0:62395:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764743.858334:0:62395:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24db0, osc lock ffff8b238a3f9300, flags 0x400000400001002 00010000:00010000:2.0:1775764743.858342:0:62395:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764743.858396:0:62395:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00000020:00200000:2.0:1775764743.858406:0:62395:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764743.858409:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764743.858416:0:62395:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00000020:00200000:2.0:1775764743.858432:0:62395:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764743.858440:0:62395:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x9:0x0] updating i_size 14 i_blocks 1 00000020:00010000:2.0:1775764743.858445:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027e910 (ffff8b23812ecbd8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764743.858448:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027e910 (ffff8b23812ecbd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764743.858450:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24db0 (ffff8b23812ec368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764743.858478:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027e910 (ffff8b23812ecbd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764743.858480:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24db0 (ffff8b23812ecbd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764743.858490:0:62395:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:14 00000020:00010000:2.0:1775764743.858498:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36bc68 (ffff8b23812ecbd8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764743.858501:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36bc68 (ffff8b23812ecbd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764743.858504:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24830 (ffff8b23812ec368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764743.858507:0:62395:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764743.858516:0:62395:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00000020:00010000:2.0:1775764743.858528:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36bc68 (ffff8b23812ecbd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764743.858531:0:62395:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24830 (ffff8b23812ecbd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764743.858536:0:62395:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1775764743.858563:0:62395:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 5 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764743.858567:0:62395:0:(file.c:2206:ll_file_io_generic()) f23.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1775764743.858570:0:62395:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:2.0:1775764743.858802:0:62395:0:(file.c:447:ll_file_release()) START file f23.sanity-pcc:[0x200003ab4:0x9:0x0](ffff8b23899c1988), flags 100000 00000002:00100000:2.0:1775764743.859735:0:62395:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea4000 x1862022098610304/t34359739163(34359739163) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764759 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764743.859750:0:62395:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764743.859756:0:62395:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6680 x1862022098611072/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000080:00200000:2.0:1775764744.062813:0:62405:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764744.062842:0:62405:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764744.062864:0:62405:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764744.062882:0:62405:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764744.062891:0:62405:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764744.062903:0:62405:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764744.062906:0:62405:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bd7d 00000080:00010000:2.0:1775764744.062914:0:62405:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd7d from it ffffa60b01f4fbb0 00010000:00010000:2.0:1775764744.062918:0:62405:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764744.062927:0:62405:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1775764744.062946:0:62405:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764744.062957:0:62405:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764744.062996:0:62405:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffcc10987a0 00000100:00100000:2.0:1775764744.063068:0:62405:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62405:1862022098611200:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764744.064541:0:62405:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1d240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764744.092262:0:62405:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62405:1862022098611200:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764744.092349:0:62405:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:2.0:1775764744.173522:0:62406:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764744.173548:0:62406:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764744.173566:0:62406:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764744.173583:0:62406:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1775764744.173590:0:62406:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764744.173598:0:62406:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764744.173601:0:62406:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bd7d 00000080:00010000:2.0:1775764744.173608:0:62406:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd7d from it ffffa60b01de3bf8 00010000:00010000:2.0:1775764744.173612:0:62406:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764744.173624:0:62406:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1775764744.173638:0:62406:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764744.173652:0:62406:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b23ab770f00, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1775764744.173658:0:62406:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open 00000002:00010000:2.0:1775764744.173672:0:62406:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f23.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:2.0:1775764744.173720:0:62406:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bda7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 62406 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.173729:0:62406:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bda7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62406 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764744.173740:0:62406:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764744.173752:0:62406:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6680 x1862022098611456/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764744.173761:0:62406:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bda7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62406 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764744.173784:0:62406:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62406:1862022098611456:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764744.173956:0:62406:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1d240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764744.194780:0:62406:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62406:1862022098611456:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764744.194803:0:62406:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bda7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62406 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.194817:0:62406:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bda7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 62406 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.194825:0:62406:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bda7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 62406 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.194832:0:62406:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bda7 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 62406 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.194840:0:62406:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bda7 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 62406 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.194846:0:62406:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6eae00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764744.194851:0:62406:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bda7 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 62406 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.194858:0:62406:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bda7 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 62406 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.194866:0:62406:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bda7 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 62406 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.194874:0:62406:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764744.194880:0:62406:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bda7 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 62406 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.194886:0:62406:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bda7 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 62406 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764744.194903:0:62406:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8b23b9ea6680 x1862022098611456/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/608 e 0 to 0 dl 1775764760 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764744.194918:0:62406:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764744.195043:0:62406:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764744.195064:0:62406:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764744.195077:0:62406:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764744.195088:0:62406:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1775764744.195095:0:62406:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764744.195099:0:62406:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764744.195102:0:62406:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bd7d 00000080:00010000:2.0:1775764744.195107:0:62406:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd7d from it ffffa60b01de3bb0 00010000:00010000:2.0:1775764744.195110:0:62406:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764744.195117:0:62406:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1775764744.195132:0:62406:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764744.195142:0:62406:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764744.195175:0:62406:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086915 arg=20a63d0 00000100:00100000:2.0:1775764744.195231:0:62406:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62406:1862022098611584:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764744.195325:0:62406:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1d240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764744.222835:0:62406:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62406:1862022098611584:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764744.223149:0:62406:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000100:00100000:2.0:1775764744.275263:0:62407:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23890d3480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764744.290215:0:62407:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job truncate:fcf15866-4739-4595-8fe1-44abde7ee9d7:62407:1862022098612096:192.168.202.128@tcp:101:truncate.0 00010000:00010000:2.0:1775764744.290247:0:62407:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17bdb5 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051df4 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.290263:0:62407:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17bdb5 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051df4 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764744.290277:0:62407:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23864f2a00 x1862022098612096/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/600 e 0 to 0 dl 1775764760 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'truncate.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764744.290288:0:62407:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17bdb5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051df4 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764744.290301:0:62407:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17bdb5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051df4 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.290314:0:62407:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/9 (0 0) 00000080:00200000:2.0:1775764744.290319:0:62407:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x9:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764744.290335:0:62407:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x9:0x0](ffff8b23899c1988) for lock 0x73cfd86cfc17bdb5 00000080:00010000:2.0:1775764744.290342:0:62407:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bdb5 from it ffffa60b01f57a58 00010000:00010000:2.0:1775764744.290346:0:62407:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17bdb5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051df4 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.290357:0:62407:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17bdb5 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051df4 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764744.290389:0:62407:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17bdb5 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051df4 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764744.290400:0:62407:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17bdb5 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051df4 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764744.290412:0:62407:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x9:0x0](000000007ac2eb40) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17bdb5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051df4 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764744.290528:0:62407:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x9:0x0]: layout version change: 4294967294 -> 5 00010000:00010000:2.0:1775764744.290556:0:62407:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17bdb5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051df4 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764744.290564:0:62407:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17bdb5 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051df4 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764744.290601:0:62407:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764744.290611:0:62407:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 5 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764744.290615:0:62407:0:(vvp_io.c:355:vvp_io_fini()) [0x200003ab4:0x9:0x0] write layout, type 3 [0x0, 0x0) 00000080:00010000:2.0:1775764744.290621:0:62407:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x9:0x0](ffff8b23899c1988) 00000002:00010000:2.0:1775764744.290630:0:62407:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x9:0x0]) in obj [0x200003ab4:0x9:0x0], intent: layout flags 02 00010000:00010000:2.0:1775764744.290645:0:62407:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17bdb5 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03051df4 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764744.290667:0:62407:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x9:0x0]: losing layout lock 00010000:00010000:2.0:1775764744.290699:0:62407:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17bdb5 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051df4 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764744.290705:0:62407:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764744.290708:0:62407:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17bdb5 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051df4 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764744.290736:0:62407:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 62407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.290743:0:62407:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62407 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764744.290753:0:62407:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764744.290759:0:62407:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6680 x1862022098612352/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/880 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764744.290781:0:62407:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62407 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764744.290800:0:62407:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job truncate:fcf15866-4739-4595-8fe1-44abde7ee9d7:62407:1862022098612352:192.168.202.128@tcp:101:truncate.0 00000100:00100000:2.0:1775764744.290868:0:62407:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1d240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764744.296134:0:62407:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job truncate:fcf15866-4739-4595-8fe1-44abde7ee9d7:62407:1862022098612352:192.168.202.128@tcp:101:truncate.0 00010000:00010000:2.0:1775764744.296159:0:62407:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 4/0,1 mode: --/EX res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.296171:0:62407:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 4/0,1 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.296183:0:62407:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.296195:0:62407:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.296208:0:62407:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.296221:0:62407:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.296228:0:62407:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764744.296239:0:62407:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea6680 x1862022098612352/t34359739168(34359739168) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/600 e 0 to 0 dl 1775764760 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'truncate.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764744.296260:0:62407:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764744.296272:0:62407:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.296283:0:62407:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/9 (0 0) 00000080:00200000:2.0:1775764744.296289:0:62407:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x9:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764744.296297:0:62407:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x9:0x0](ffff8b23899c1988) for lock 0x73cfd86cfc17bdbc 00000080:00010000:2.0:1775764744.296314:0:62407:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bdbc from it ffffa60b01f57b40 00010000:00010000:2.0:1775764744.296318:0:62407:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.296326:0:62407:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764744.296346:0:62407:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 5 restore needed 0 00010000:00010000:2.0:1775764744.296358:0:62407:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764744.296385:0:62407:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764744.296394:0:62407:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x9:0x0](000000007ac2eb40) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764744.296505:0:62407:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764744.296514:0:62407:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764744.296540:0:62407:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764744.296548:0:62407:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 10, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764744.296551:0:62407:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764744.296557:0:62407:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764744.296596:0:62407:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764744.296667:0:62407:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764744.296676:0:62407:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764744.296684:0:62407:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x9:0x0](000000007ac2eb40) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764744.296715:0:62407:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764744.296720:0:62407:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 10, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764744.296722:0:62407:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764744.296724:0:62407:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764744.296751:0:62407:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:2.0:1775764744.296779:0:62407:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 14 blocks 1 [cma]time: 1775764743 1775764743 1775764743 by oinfo size 14 blocks 1 [cma]time 1775764743 1775764743 1775764743 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00000080:00200000:2.0:1775764744.296832:0:62407:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x9:0x0]: layout version change: 4294967294 -> 6 00010000:00010000:2.0:1775764744.296841:0:62407:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764744.296849:0:62407:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bdbc lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051e02 expref: -99 pid: 62407 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00020000:00200000:2.0:1775764744.296860:0:62407:0:(lov_io.c:308:lov_io_mirror_write_intent()) [0x200003ab4:0x9:0x0]there are 0 components to be staled to modify file extent [0x0, 0xffffffffffffffff), iot: 3 00020000:00200000:2.0:1775764744.296863:0:62407:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764744.296866:0:62407:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764744.296869:0:62407:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764744.296883:0:62407:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:2.0:1775764744.296893:0:62407:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764744.296897:0:62407:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e241b0, osc lock ffff8b238a3f9c80, flags 0x0 00010000:00010000:2.0:1775764744.297153:0:62407:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1775764744.297164:0:62407:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1775764744.297175:0:62407:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00000008:00010000:2.0:1775764744.297185:0:62407:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 14, setting rss=14; kms=14, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1775764744.297195:0:62407:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1775764744.297203:0:62407:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00000080:00200000:2.0:1775764744.301132:0:62407:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x9:0x0] updating i_size 256 i_blocks 1 00000020:00010000:2.0:1775764744.301139:0:62407:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36bc68 (ffff8b23812ecbd8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764744.301143:0:62407:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36bc68 (ffff8b23812ecbd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764744.301147:0:62407:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e241b0 (ffff8b23812ec368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764744.301156:0:62407:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1775764744.301172:0:62407:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 15 lru_type: 1 00000020:00010000:2.0:1775764744.301185:0:62407:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36bc68 (ffff8b23812ecbd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764744.301188:0:62407:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e241b0 (ffff8b23812ecbd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764744.301227:0:62407:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1775764744.301250:0:62407:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file f23.sanity-pcc:[0x200003ab4:0x9:0x0](ffff8b23899c1988) current size 256, valid attrs 168, mode ffff, uid -1503366868, gid -1, new size 256, atime 1775764744.000000000, mtime 1775764744.000000000, ctime 1775764744.000000000, rc 0 00000080:00200000:1.0:1775764744.326034:0:62408:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764744.326063:0:62408:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764744.326082:0:62408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1775764744.326099:0:62408:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:1.0:1775764744.326106:0:62408:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764744.326115:0:62408:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764744.326119:0:62408:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bd7d 00000080:00010000:1.0:1775764744.326125:0:62408:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd7d from it ffffa60b01f4fbf8 00010000:00010000:1.0:1775764744.326128:0:62408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1775764744.326136:0:62408:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1775764744.326150:0:62408:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764744.326164:0:62408:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b2388011b00, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764744.326169:0:62408:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open 00000002:00010000:1.0:1775764744.326182:0:62408:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f23.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764744.326231:0:62408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bdc3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 62408 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764744.326238:0:62408:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bdc3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62408 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764744.326249:0:62408:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764744.326261:0:62408:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ff800 x1862022098612736/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764744.326275:0:62408:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bdc3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62408 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764744.326297:0:62408:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62408:1862022098612736:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764744.330628:0:62408:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23894b6780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764744.340131:0:62408:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62408:1862022098612736:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764744.340156:0:62408:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bdc3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62408 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.340170:0:62408:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bdc3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 62408 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.340178:0:62408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bdc3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 62408 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.340185:0:62408:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bdc3 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 62408 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.340194:0:62408:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bdc3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 62408 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.340200:0:62408:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8741800 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764744.340205:0:62408:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bdc3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 62408 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.340212:0:62408:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bdc3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 62408 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.340220:0:62408:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bdc3 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 62408 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.340228:0:62408:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764744.340234:0:62408:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bdc3 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 62408 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.340245:0:62408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bdc3 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 62408 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764744.340268:0:62408:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8b23864ff800 x1862022098612736/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/608 e 0 to 0 dl 1775764760 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764744.340283:0:62408:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764744.340502:0:62408:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764744.340524:0:62408:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764744.340539:0:62408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764744.340552:0:62408:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:2.0:1775764744.340559:0:62408:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764744.340565:0:62408:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764744.340568:0:62408:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bd7d 00000080:00010000:2.0:1775764744.340574:0:62408:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd7d from it ffffa60b01f4fbb0 00010000:00010000:2.0:1775764744.340577:0:62408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764744.340585:0:62408:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:2.0:1775764744.340601:0:62408:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764744.340612:0:62408:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764744.340644:0:62408:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086915 arg=c373d0 00000100:00100000:2.0:1775764744.340700:0:62408:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62408:1862022098612864:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764744.340783:0:62408:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1d240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764744.356179:0:62408:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62408:1862022098612864:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764744.356523:0:62408:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:2.0:1775764744.515235:0:62418:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764744.515266:0:62418:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764744.515285:0:62418:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764744.515301:0:62418:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:2.0:1775764744.515310:0:62418:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764744.515318:0:62418:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764744.515322:0:62418:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bd7d 00000080:00010000:2.0:1775764744.515328:0:62418:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bd7d from it ffffa60b01c13bb0 00010000:00010000:2.0:1775764744.515332:0:62418:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764744.515340:0:62418:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17bd7d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ce3 expref: -99 pid: 62372 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:2.0:1775764744.515354:0:62418:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764744.515361:0:62418:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764744.515383:0:62418:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7fffef249460 00000100:00100000:2.0:1775764744.515490:0:62418:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62418:1862022098612992:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764744.519860:0:62418:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1de40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764744.538192:0:62418:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62418:1862022098612992:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764744.538276:0:62418:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000100:00100000:1.0:1775764744.766275:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200000:2.0:1775764744.803900:0:62071:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:2.0:1775764744.803925:0:62071:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 4 00000008:00010000:2.0:1775764744.803930:0:62071:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e241b0, osc lock ffff8b238a3f9c80, flags 0x80000000 00010000:00010000:2.0:1775764744.803957:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1775764744.803978:0:62071:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1775764744.803992:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1775764744.804001:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1775764744.804009:0:62071:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1775764744.820716:0:62071:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x9:0x0] updating i_size 0 i_blocks 11 00000020:00010000:2.0:1775764744.820723:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384cd68a8 (ffff8b238a3ff2d8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764744.820726:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384cd68a8 (ffff8b238a3ff2d8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764744.820731:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e241b0 (ffff8b23812ecbd8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764744.820742:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1775764744.820761:0:62071:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00000020:00010000:2.0:1775764744.820775:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384cd68a8 (ffff8b238a3ff2d8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764744.820777:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e241b0 (ffff8b238a3ff2d8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764744.820833:0:62071:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1775764744.820859:0:62071:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file f23.sanity-pcc:[0x200003ab4:0x9:0x0](ffff8b23899c1988) current size 0, valid attrs a168, mode 0, uid 2, gid 0, new size 0, atime 1775764744.000000000, mtime 1775764744.000000000, ctime 1775764744.000000000, rc 0 00000080:00200008:2.0:1775764744.820938:0:62071:0:(file.c:2612:do_file_write_iter()) START file f23.sanity-pcc:[0x200003ab4:0x9:0x0], ppos: 0, count: 15 00000080:00200000:2.0:1775764744.820951:0:62071:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 15 bytes 00000080:00200000:2.0:1775764744.820958:0:62071:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 15 00000080:00200000:2.0:1775764744.820981:0:62071:0:(file.c:2058:ll_file_io_generic()) f23.sanity-pcc: write ppos: 0, bytes: 15 00000080:00200000:2.0:1775764744.820986:0:62071:0:(file.c:1815:ll_io_set_mirror()) f23.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764744.820993:0:62071:0:(cl_io.c:214:cl_io_rw_init()) header@000000002e91068a[0x110, 1, [0x200003ab4:0x9:0x0] hash] 00000020:00200000:2.0:1775764744.820994:0:62071:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 15) 0 0 00000080:00200000:2.0:1775764744.820998:0:62071:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764744.821012:0:62071:0:(lov_io.c:308:lov_io_mirror_write_intent()) [0x200003ab4:0x9:0x0]there are 0 components to be staled to modify file extent [0x0, 0xffffffffffffffff), iot: 2 00020000:00200000:2.0:1775764744.821017:0:62071:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764744.821019:0:62071:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:2.0:1775764744.821024:0:62071:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:2.0:1775764744.821032:0:62071:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 15) 15, 15 00020000:00200000:2.0:1775764744.821035:0:62071:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764744.821061:0:62071:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1775764744.821073:0:62071:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764744.821077:0:62071:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e241b0, osc lock ffff8b238a3f9c80, flags 0x0 00010000:00010000:2.0:1775764744.821096:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:2.0:1775764744.821113:0:62071:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:2.0:1775764744.821124:0:62071:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:2.0:1775764744.821132:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:2.0:1775764744.821140:0:62071:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00000080:00200000:2.0:1775764744.821150:0:62071:0:(vvp_io.c:1286:vvp_io_write_start()) f23.sanity-pcc: write [0, 15) append 0 append lockpos 0 00000080:00200000:2.0:1775764744.821158:0:62071:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 15 bytes 00000080:00200000:2.0:1775764744.821200:0:62071:0:(rw26.c:858:ll_write_end()) pos 0, len 15, copied 15 00000080:00200000:2.0:1775764744.821206:0:62071:0:(rw26.c:916:ll_write_end()) page@00000000d1ce3a9a[3 000000007b9b600d 2 1 0000000024d4832d] 00000080:00200000:2.0:1775764744.821209:0:62071:0:(rw26.c:916:ll_write_end()) vmpage @000000000da42d4a 17ffffc0001001 3:0 ffff8b23baeaec40 0 lru 00000080:00200000:2.0:1775764744.821215:0:62071:0:(rw26.c:916:ll_write_end()) osc-page@00000000c66b2839 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 000000003596ffcd 00000000336aac03 > 3< 0 0 > 4< 0 0 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764744.821217:0:62071:0:(rw26.c:916:ll_write_end()) end page@00000000d1ce3a9a 00000080:00200000:2.0:1775764744.821218:0:62071:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764744.821222:0:62071:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 15 prio 0 00000008:00010000:2.0:1775764744.821238:0:62071:0:(osc_cache.c:848:osc_extent_find()) ### extent: 000000008a7bf9d4 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00000080:00200000:2.0:1775764744.821255:0:62071:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b23899c1b38, count 1, dirtied 1 00000080:00200000:2.0:1775764744.821259:0:62071:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 15 bytes, tot: 15 00000080:00200000:2.0:1775764744.821262:0:62071:0:(vvp_io.c:1353:vvp_io_write_start()) f23.sanity-pcc: write bytes 0, result: 15 00000020:00010000:2.0:1775764744.821276:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36bc68 (ffff8b23812ecbd8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764744.821278:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36bc68 (ffff8b23812ecbd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764744.821281:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e241b0 (ffff8b23812ec368/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764744.821285:0:62071:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:2.0:1775764744.821293:0:62071:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00000020:00010000:2.0:1775764744.821311:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36bc68 (ffff8b23812ecbd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764744.821314:0:62071:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e241b0 (ffff8b23812ecbd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764744.821319:0:62071:0:(vvp_io.c:517:vvp_io_advance()) advancing 15 bytes 00000080:00200000:2.0:1775764744.821323:0:62071:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:2.0:1775764744.821337:0:62071:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764744.821341:0:62071:0:(file.c:2206:ll_file_io_generic()) f23.sanity-pcc: 2 io complete with rc: 0, result: 15, restart: 0 00000080:00200000:2.0:1775764744.821347:0:62071:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x9:0x0] updating i_size 15 i_blocks 11 00000080:00200000:2.0:1775764744.821350:0:62071:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 15 00000080:00200008:2.0:1775764744.821375:0:62071:0:(file.c:447:ll_file_release()) START file f23.sanity-pcc:[0x200003ab4:0x9:0x0](ffff8b23899c1988), flags 100001 00010000:00010000:2.0:1775764744.821543:0:62071:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17bdd1 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e6b expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1775764744.844706:0:62426:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62426:1862022098614144:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764744.844744:0:62426:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.844770:0:62426:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.844789:0:62426:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.844802:0:62426:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.844817:0:62426:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.844827:0:62426:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.844834:0:62426:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764744.844847:0:62426:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b2382dad880 x1862022098614144/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764760 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764744.844862:0:62426:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.844876:0:62426:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764744.844883:0:62426:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764744.844942:0:62426:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764744.844950:0:62426:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bddf 00000080:00010000:2.0:1775764744.844956:0:62426:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764744.844960:0:62426:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bddf from it ffffa60b01f57bf8 00010000:00010000:2.0:1775764744.844964:0:62426:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.844975:0:62426:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764744.845007:0:62426:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764744.845038:0:62426:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b23a00a7e00, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1775764744.845046:0:62426:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open 00000002:00010000:2.0:1775764744.845072:0:62426:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f23.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:2.0:1775764744.845128:0:62426:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bde6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 62426 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.845135:0:62426:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bde6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62426 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764744.845146:0:62426:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764744.845152:0:62426:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7800 x1862022098614400/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764744.845162:0:62426:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bde6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62426 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764744.845186:0:62426:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62426:1862022098614400:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764744.846128:0:62426:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1de40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764744.861990:0:62426:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62426:1862022098614400:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764744.862020:0:62426:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bde6 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62426 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.862039:0:62426:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bde6 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 62426 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.862045:0:62426:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bde6 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 62426 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.862052:0:62426:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bde6 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 62426 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.862061:0:62426:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bde6 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 62426 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.862066:0:62426:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6eb200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764744.862070:0:62426:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bde6 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 62426 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.862076:0:62426:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bde6 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 62426 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.862083:0:62426:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bde6 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 62426 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.862090:0:62426:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764744.862095:0:62426:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bde6 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 62426 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764744.862101:0:62426:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17bde6 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 62426 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764744.862118:0:62426:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8b23b9ea7800 x1862022098614400/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/608 e 0 to 0 dl 1775764760 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764744.862136:0:62426:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764744.862276:0:62426:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764744.862295:0:62426:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764744.862728:0:62426:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764744.862743:0:62426:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764744.862750:0:62426:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764744.862768:0:62426:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764744.862771:0:62426:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bddf 00000080:00010000:2.0:1775764744.862778:0:62426:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bddf from it ffffa60b01f57bb0 00010000:00010000:2.0:1775764744.862781:0:62426:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764744.862790:0:62426:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764744.862804:0:62426:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764744.862815:0:62426:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764744.862851:0:62426:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086915 arg=25503d0 00000100:00100000:2.0:1775764744.862910:0:62426:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62426:1862022098614528:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764744.863002:0:62426:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1de40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764744.867314:0:62426:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62426:1862022098614528:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764744.867631:0:62426:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000100:00100000:1.0:1775764744.996118:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119053568 00000100:00100000:1.0:1775764744.996129:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119053568 00000100:00100000:1.0:1775764744.996237:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 251 00000100:00100000:1.0:1775764744.996247:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dafb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119053568:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764744.996265:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17bdd1 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03051e6b expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764744.996356:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764744.996363:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dafb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119053568:12345-192.168.202.128@tcp:104: Request processed in 118us (319us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764744.996370:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 251 00010000:00010000:2.0:1775764744.996477:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764744.996485:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17bdd1 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03051e6b expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764744.996494:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5f200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764744.996498:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17bdd1 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03051e6b expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764744.996506:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17bdd1 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03051e6b expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764744.996515:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17bdd1 lrc: 3/0,0 mode: CW/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x57b8ff0e03051e6b expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000002:00100000:2.0:1775764744.996579:0:46150:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864f1180 x1862022098613376/t34359739169(34359739169) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764760 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764744.996590:0:46150:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764744.996596:0:46150:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7800 x1862022098614912/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764744.996621:0:46150:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_02:fcf15866-4739-4595-8fe1-44abde7ee9d7:46150:1862022098614912:192.168.202.128@tcp:35:ldlm_bl.0 00000100:00100000:2.0:1775764744.996677:0:46150:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1de40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764745.002782:0:46150:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_02:fcf15866-4739-4595-8fe1-44abde7ee9d7:46150:1862022098614912:192.168.202.128@tcp:35:ldlm_bl.0 00000002:00100000:2.0:1775764745.002818:0:46150:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864f1180 x1862022098613376/t34359739169(34359739169) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764760 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764745.002897:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17bdd1 lrc: 2/0,0 mode: --/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03051e6b expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764745.002912:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764745.002951:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17bdd1 lrc: 1/0,0 mode: --/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03051e6b expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764745.002960:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17bdd1 lrc: 0/0,0 mode: --/CW res: [0x200003ab4:0x9:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03051e6b expref: -99 pid: 62071 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764745.002976:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764745.002978:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764745.002979:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764745.002987:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764745.003004:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098615040:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764745.011238:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098615040:192.168.202.128@tcp:103:ldlm_bl.0 00000080:00200000:2.0:1775764745.034046:0:62432:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764745.034138:0:62432:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17bdfb lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051efe expref: -99 pid: 62432 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764745.034152:0:62432:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17bdfb lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051efe expref: -99 pid: 62432 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764745.034161:0:62432:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x9:0x0](000000007ac2eb40) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17bdfb lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051efe expref: -99 pid: 62432 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764745.034227:0:62432:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764745.034237:0:62432:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764745.034240:0:62432:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764745.034247:0:62432:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764745.034286:0:62432:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:2.0:1775764745.034316:0:62432:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 14 blocks 1 [cma]time: 1775764743 1775764743 1775764743 by oinfo size 15 blocks 11 [cma]time 1775764744 1775764744 1775764743 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00000080:00200000:2.0:1775764745.034386:0:62432:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x9:0x0]: layout version change: 4294967294 -> 7 00010000:00010000:2.0:1775764745.034400:0:62432:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17bdfb lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051efe expref: -99 pid: 62432 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764745.034418:0:62432:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17bdfb lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051efe expref: -99 pid: 62432 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:2.0:1775764745.034916:0:62432:0:(file.c:2454:do_file_read_iter()) START file f23.sanity-pcc:[0x200003ab4:0x9:0x0], ppos: 0, count: 1048576 00000080:00200000:2.0:1775764745.034943:0:62432:0:(rw.c:1928:ll_readpage()) fast read pgno: 0 00000080:00200000:2.0:1775764745.034964:0:62432:0:(file.c:2058:ll_file_io_generic()) f23.sanity-pcc: read ppos: 0, bytes: 1048576 00000080:00200000:2.0:1775764745.034970:0:62432:0:(file.c:1815:ll_io_set_mirror()) f23.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764745.034977:0:62432:0:(cl_io.c:214:cl_io_rw_init()) header@000000002e91068a[0x110, 1, [0x200003ab4:0x9:0x0] hash] 00000020:00200000:2.0:1775764745.034979:0:62432:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 1048576) 0 0 00000080:00200000:2.0:1775764745.034981:0:62432:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764745.034992:0:62432:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 10, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764745.034995:0:62432:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764745.035001:0:62432:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:2.0:1775764745.035004:0:62432:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764745.035024:0:62432:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:2.0:1775764745.035036:0:62432:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764745.035041:0:62432:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24db0, osc lock ffff8b238a3f9c80, flags 0x400000000000000 00010000:00010000:2.0:1775764745.035056:0:62432:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00010000:00010000:2.0:1775764745.035067:0:62432:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00010000:00010000:2.0:1775764745.035078:0:62432:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00000008:00010000:2.0:1775764745.035088:0:62432:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 15, setting rss=15; kms=15, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00010000:00010000:2.0:1775764745.035098:0:62432:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00010000:00010000:2.0:1775764745.035105:0:62432:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00000080:00200000:2.0:1775764745.035115:0:62432:0:(vvp_io.c:826:vvp_io_read_start()) f23.sanity-pcc: read [0, 1048576) 00000001:00010000:2.0:1775764745.035128:0:62432:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x9:0x0] 00000008:00010000:2.0:1775764745.035136:0:62432:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764745.035138:0:62432:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e244b0, osc lock ffff8b238a3f9300, flags 0x400000400001002 00010000:00010000:2.0:1775764745.035145:0:62432:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00010000:00010000:2.0:1775764745.035153:0:62432:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00000020:00200000:2.0:1775764745.035161:0:62432:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764745.035164:0:62432:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00010000:00010000:2.0:1775764745.035176:0:62432:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00000020:00200000:2.0:1775764745.035181:0:62432:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764745.035187:0:62432:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x9:0x0] updating i_size 15 i_blocks 11 00000020:00010000:2.0:1775764745.035190:0:62432:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027e910 (ffff8b23812ecbd8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764745.035192:0:62432:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027e910 (ffff8b23812ecbd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764745.035195:0:62432:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e244b0 (ffff8b23812ec368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764745.035199:0:62432:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027e910 (ffff8b23812ecbd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764745.035201:0:62432:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e244b0 (ffff8b23812ecbd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764745.035208:0:62432:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:0 00000008:00010000:2.0:1775764745.035243:0:62432:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b2383c9e840 00000008:00010000:2.0:1775764745.035285:0:46654:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b2383c9e840 00010000:00010000:2.0:1775764745.035304:0:46654:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00000008:00100000:2.0:1775764745.035339:0:46654:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea7800 - object 0x240000400:282 offset 0<>4096 00000100:00100000:2.0:1775764745.035389:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764745.035405:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098615808:192.168.202.128@tcp:3:lfs.0 00000008:00010000:2.0:1775764745.042293:0:2379:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8b2383c9e840 00000020:00200000:2.0:1775764745.042320:0:2379:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1775764745.042342:0:2379:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8b2383c9e840 00000100:00100000:2.0:1775764745.042384:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098615808:192.168.202.128@tcp:3:lfs.0 00000080:00200000:2.0:1775764745.196101:0:62443:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764745.196132:0:62443:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764745.196153:0:62443:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764745.196173:0:62443:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764745.196181:0:62443:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764745.196190:0:62443:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764745.196194:0:62443:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bddf 00000080:00010000:2.0:1775764745.196200:0:62443:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bddf from it ffffa60b01f57bf8 00010000:00010000:2.0:1775764745.196204:0:62443:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764745.196213:0:62443:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764745.196228:0:62443:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764745.196246:0:62443:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b23ab771b00, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1775764745.196253:0:62443:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open 00000002:00010000:2.0:1775764745.196269:0:62443:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f23.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:2.0:1775764745.196550:0:62443:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be02 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 62443 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.196560:0:62443:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be02 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62443 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764745.196573:0:62443:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764745.196588:0:62443:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7800 x1862022098616448/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764745.196604:0:62443:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be02 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62443 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764745.196631:0:62443:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62443:1862022098616448:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764745.196931:0:62443:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1de40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764745.201910:0:62443:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62443:1862022098616448:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764745.201935:0:62443:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be02 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62443 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.201950:0:62443:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be02 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 62443 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.201956:0:62443:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be02 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 62443 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.201964:0:62443:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be02 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 62443 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.201973:0:62443:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be02 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 62443 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.201980:0:62443:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6eb200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764745.201986:0:62443:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be02 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 62443 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.201993:0:62443:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be02 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 62443 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.202000:0:62443:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be02 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 62443 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.202010:0:62443:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764745.202017:0:62443:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be02 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 62443 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.202025:0:62443:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be02 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 62443 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764745.202045:0:62443:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8b23b9ea7800 x1862022098616448/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/608 e 0 to 0 dl 1775764761 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764745.202060:0:62443:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764745.202189:0:62443:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764745.202211:0:62443:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764745.202225:0:62443:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764745.202237:0:62443:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764745.202244:0:62443:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764745.202251:0:62443:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764745.202254:0:62443:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bddf 00000080:00010000:2.0:1775764745.202259:0:62443:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bddf from it ffffa60b01f57bb0 00010000:00010000:2.0:1775764745.202262:0:62443:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764745.202270:0:62443:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1775764745.202290:0:62443:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764745.202313:0:62443:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764745.202353:0:62443:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086915 arg=14493d0 00000100:00100000:2.0:1775764745.202433:0:62443:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62443:1862022098616576:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764745.202550:0:62443:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1de40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764745.206059:0:62443:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62443:1862022098616576:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764745.206352:0:62443:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:2.0:1775764745.254102:0:62449:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764745.254131:0:62449:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764745.254149:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764745.254167:0:62449:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1775764745.254176:0:62449:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764745.254184:0:62449:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764745.254188:0:62449:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bddf 00000080:00010000:2.0:1775764745.254195:0:62449:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bddf from it ffffa60b01fdfbf8 00010000:00010000:2.0:1775764745.254200:0:62449:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764745.254209:0:62449:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1775764745.254229:0:62449:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764745.254245:0:62449:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764745.254251:0:62449:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764745.254256:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764745.254264:0:62449:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1775764745.254270:0:62449:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764745.254274:0:62449:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764745.254277:0:62449:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bddf 00000080:00010000:2.0:1775764745.254281:0:62449:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bddf from it ffffa60b01fdfb38 00010000:00010000:2.0:1775764745.254283:0:62449:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764745.254291:0:62449:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1775764745.254589:0:62449:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:2.0:1775764745.254594:0:62449:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b2386296000, open_flags 8042, mode 81a4 opened 0 00000080:00200000:2.0:1775764745.254604:0:62449:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000002:00010000:2.0:1775764745.254626:0:62449:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f23.sanity-pcc,[0x200003ab4:0xa:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0100103 00010000:00010000:2.0:1775764745.254639:0:62449:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1775764745.254697:0:62449:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1775764745.254705:0:62449:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764745.254707:0:62449:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bddf lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051e8e expref: -99 pid: 62426 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1775764745.254740:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 62449 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.254748:0:62449:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62449 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764745.254773:0:62449:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764745.254786:0:62449:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7800 x1862022098616832/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764745.254795:0:62449:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62449 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764745.254820:0:62449:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:62449:1862022098616832:192.168.202.128@tcp:101:multiop.0 00000100:00100000:2.0:1775764745.255747:0:62449:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1de40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764745.262180:0:62449:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:62449:1862022098616832:192.168.202.128@tcp:101:multiop.0 00010000:00010000:2.0:1775764745.262207:0:62449:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.262230:0:62449:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.262243:0:62449:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.262253:0:62449:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.262264:0:62449:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.262276:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.262283:0:62449:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.262289:0:62449:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764745.262644:0:62449:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=2b, status=0 req@ffff8b23b9ea7800 x1862022098616832/t34359739176(34359739176) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/864 e 0 to 0 dl 1775764761 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764745.262658:0:62449:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7800 x1862022098616832/t34359739176(34359739176) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/864 e 0 to 0 dl 1775764761 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764745.262671:0:62449:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764745.262918:0:62449:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.262932:0:62449:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/9 (0 0) 00000080:00200000:2.0:1775764745.262940:0:62449:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764745.263031:0:62449:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0x9:0x0](ffff8b23899c1988): rc = 0 00000080:00200000:2.0:1775764745.263051:0:62449:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x9:0x0]: layout version change: 7 -> 7 00000080:00010000:2.0:1775764745.263060:0:62449:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x9:0x0](ffff8b23899c1988) for lock 0x73cfd86cfc17be09 00000080:00010000:2.0:1775764745.263071:0:62449:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x9:0x0](ffff8b23899c1988) 00000080:00200008:2.0:1775764745.263084:0:62449:0:(file.c:1057:ll_file_open()) START file f23.sanity-pcc:[0x200003ab4:0x9:0x0](ffff8b23899c1988), flags 100102 00000002:00100000:2.0:1775764745.263103:0:62449:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7800 x1862022098616832/t34359739176(34359739176) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764761 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764745.263136:0:62449:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be09 from it ffff8b238504c0c0 00010000:00010000:2.0:1775764745.263141:0:62449:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.263150:0:62449:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764745.263180:0:62449:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x9:0x0](ffff8b23899c1988), inode mode 81a4 mask 46 00000080:00200008:2.0:1775764745.263213:0:62449:0:(file.c:6226:ll_getattr_dentry()) START file f23.sanity-pcc:[0x200003ab4:0x9:0x0](ffff8b23899c1988), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764745.263219:0:62449:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x9:0x0](ffff8b23899c1988),name=f23.sanity-pcc 00000002:00010000:2.0:1775764745.263240:0:62449:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x9:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764745.263249:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764745.263259:0:62449:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764745.263271:0:62449:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764745.263275:0:62449:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x9:0x0](ffff8b23899c1988) 00000080:00010000:2.0:1775764745.263278:0:62449:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x9:0x0](ffff8b23899c1988) for lock 0x73cfd86cfc17be09 00000080:00010000:2.0:1775764745.263281:0:62449:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be09 from it ffffa60b01fdfcd8 00010000:00010000:2.0:1775764745.263284:0:62449:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764745.263291:0:62449:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00a00000:2.0:1775764745.263582:0:62449:0:(llite_mmap.c:586:ll_file_mmap()) VFS_Op: fid=[0x200003ab4:0x9:0x0] vma=ffff8b2391bafae0 start=0x7fca1340a000 end=0x7fca1340b000 vm_flags=0xfb 00000080:00200000:2.0:1775764745.264661:0:62449:0:(rw.c:1928:ll_readpage()) fast read pgno: 0 00000080:00200000:2.0:1775764745.264671:0:62449:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764745.264683:0:62449:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 10, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764745.264686:0:62449:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764745.264694:0:62449:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764745.264723:0:62449:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1775764745.264742:0:62449:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764745.264748:0:62449:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e244b0, osc lock ffff8b238a3f9300, flags 0x0 00010000:00010000:2.0:1775764745.264859:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:2.0:1775764745.264876:0:62449:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:2.0:1775764745.264889:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:2.0:1775764745.264904:0:62449:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:2.0:1775764745.264911:0:62449:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00000001:00010000:2.0:1775764745.264929:0:62449:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x9:0x0] 00000008:00010000:2.0:1775764745.264935:0:62449:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764745.264938:0:62449:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e240b0, osc lock ffff8b238a3f8850, flags 0x400001002 00010000:00010000:2.0:1775764745.264946:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:2.0:1775764745.264955:0:62449:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00000020:00200000:2.0:1775764745.264964:0:62449:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764745.264968:0:62449:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 7/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:2.0:1775764745.264975:0:62449:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00000020:00200000:2.0:1775764745.264982:0:62449:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764745.264988:0:62449:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x9:0x0] updating i_size 15 i_blocks 11 00000020:00010000:2.0:1775764745.264992:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c248 (ffff8b23812ec248/0) at cl_lock_release():245 00000020:00010000:2.0:1775764745.264995:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c248 (ffff8b23812ec248/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764745.264998:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e240b0 (ffff8b23812ecc68/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764745.265001:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c248 (ffff8b23812ec248/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764745.265004:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e240b0 (ffff8b23812ec248/1) at cl_lock_fini():72 00000008:00010000:2.0:1775764745.265047:0:62449:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b2383c9e240 00000008:00010000:2.0:1775764745.265097:0:46654:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b2383c9e240 00010000:00010000:2.0:1775764745.265120:0:46654:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00000008:00100000:2.0:1775764745.265159:0:46654:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea4380 - object 0x240000400:282 offset 0<>4096 00000020:00010000:2.0:1775764745.270024:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b308 (ffff8b23812ec248/0) at cl_lock_release():245 00000020:00010000:2.0:1775764745.270030:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b308 (ffff8b23812ec248/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764745.270034:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e244b0 (ffff8b23812ecc68/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764745.270045:0:62449:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:2.0:1775764745.270062:0:62449:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00000020:00010000:2.0:1775764745.270078:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b308 (ffff8b23812ec248/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764745.270081:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e244b0 (ffff8b23812ec248/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764745.270117:0:62449:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764745.270163:0:62449:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764745.270174:0:62449:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764745.270179:0:62449:0:(vvp_io.c:355:vvp_io_fini()) [0x200003ab4:0x9:0x0] write layout, type 5 [0x0, 0x0) 00000080:00010000:2.0:1775764745.270191:0:62449:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x9:0x0](ffff8b23899c1988) 00000002:00010000:2.0:1775764745.270204:0:62449:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x9:0x0]) in obj [0x200003ab4:0x9:0x0], intent: layout flags 02 00010000:00010000:2.0:1775764745.270227:0:62449:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764745.270262:0:62449:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x9:0x0]: losing layout lock 00010000:00010000:2.0:1775764745.270383:0:62449:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17bdfb lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03051efe expref: -99 pid: 62432 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764745.270402:0:62449:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x9:0x0]: losing layout lock 00010000:00010000:2.0:1775764745.270479:0:62449:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764745.270486:0:62449:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17bdfb lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051efe expref: -99 pid: 62432 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764745.270492:0:62449:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1775764745.270495:0:62449:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be09 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051f59 expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764745.270508:0:62449:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17bdfb lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051efe expref: -99 pid: 62432 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764745.270539:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 62449 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.270546:0:62449:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62449 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764745.270558:0:62449:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764745.270569:0:62449:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4700 x1862022098617216/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 384/880 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764745.270579:0:62449:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62449 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764745.270603:0:62449:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:62449:1862022098617216:192.168.202.128@tcp:101:multiop.0 00000100:00100000:2.0:1775764745.270774:0:62449:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1de40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764745.283702:0:62449:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:62449:1862022098617216:192.168.202.128@tcp:101:multiop.0 00010000:00010000:2.0:1775764745.283723:0:62449:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 4/0,1 mode: --/EX res: [0x200003ab4:0x9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.283735:0:62449:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 4/0,1 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.283747:0:62449:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.283758:0:62449:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.283772:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.283780:0:62449:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.283787:0:62449:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764745.283798:0:62449:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea4700 x1862022098617216/t34359739177(34359739177) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 384/600 e 0 to 0 dl 1775764761 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764745.283824:0:62449:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764745.283837:0:62449:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.283850:0:62449:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/9 (0 0) 00000080:00200000:2.0:1775764745.283855:0:62449:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x9:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764745.283862:0:62449:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x9:0x0](ffff8b23899c1988) for lock 0x73cfd86cfc17be10 00000080:00010000:2.0:1775764745.283869:0:62449:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be10 from it ffffa60b01fdfb38 00010000:00010000:2.0:1775764745.283872:0:62449:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.283881:0:62449:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764745.283915:0:62449:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00010000:00010000:2.0:1775764745.283926:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764745.283937:0:62449:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764745.283946:0:62449:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x9:0x0](000000007ac2eb40) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764745.284033:0:62449:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764745.284042:0:62449:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764745.284069:0:62449:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764745.284078:0:62449:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 10, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764745.284080:0:62449:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764745.284086:0:62449:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764745.284135:0:62449:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764745.284189:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764745.284199:0:62449:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764745.284206:0:62449:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x9:0x0](000000007ac2eb40) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764745.284239:0:62449:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764745.284243:0:62449:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 10, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764745.284246:0:62449:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764745.284248:0:62449:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764745.284278:0:62449:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:2.0:1775764745.284297:0:62449:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 15 blocks 11 [cma]time: 1775764744 1775764744 1775764743 by oinfo size 15 blocks 11 [cma]time 1775764744 1775764744 1775764745 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00000080:00200000:2.0:1775764745.284359:0:62449:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x9:0x0]: layout version change: 4294967294 -> 8 00010000:00010000:2.0:1775764745.284370:0:62449:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764745.284377:0:62449:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00020000:00200000:2.0:1775764745.284389:0:62449:0:(lov_io.c:308:lov_io_mirror_write_intent()) [0x200003ab4:0x9:0x0]there are 0 components to be staled to modify file extent [0x0, 0xffffffffffffffff), iot: 5 00020000:00200000:2.0:1775764745.284392:0:62449:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764745.284394:0:62449:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764745.284401:0:62449:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764745.284429:0:62449:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1775764745.284441:0:62449:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764745.284445:0:62449:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e243b0, osc lock ffff8b238a3f9300, flags 0x0 00010000:00010000:2.0:1775764745.284496:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764745.284507:0:62449:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764745.284517:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00000008:00010000:2.0:1775764745.284526:0:62449:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 15, setting rss=15; kms=15, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764745.284536:0:62449:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764745.284549:0:62449:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00000001:00010000:2.0:1775764745.284567:0:62449:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x9:0x0] 00000008:00010000:2.0:1775764745.284573:0:62449:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764745.284575:0:62449:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24530, osc lock ffff8b238a3f8850, flags 0x400001002 00010000:00010000:2.0:1775764745.284583:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764745.284592:0:62449:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00000020:00200000:2.0:1775764745.284601:0:62449:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764745.284604:0:62449:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 7/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764745.284611:0:62449:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00000020:00200000:2.0:1775764745.284618:0:62449:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764745.284625:0:62449:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x9:0x0] updating i_size 15 i_blocks 11 00000020:00010000:2.0:1775764745.284629:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c248 (ffff8b23812ec248/0) at cl_lock_release():245 00000020:00010000:2.0:1775764745.284632:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c248 (ffff8b23812ec248/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764745.284635:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24530 (ffff8b23812ecc68/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764745.284639:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c248 (ffff8b23812ec248/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764745.284642:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24530 (ffff8b23812ec248/1) at cl_lock_fini():72 00000020:00010000:2.0:1775764745.284650:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b308 (ffff8b23812ec248/0) at cl_lock_release():245 00000020:00010000:2.0:1775764745.284653:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b308 (ffff8b23812ec248/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764745.284655:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e243b0 (ffff8b23812ecc68/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764745.284659:0:62449:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764745.284667:0:62449:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00000020:00010000:2.0:1775764745.284676:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b308 (ffff8b23812ec248/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764745.284678:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e243b0 (ffff8b23812ec248/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764745.284694:0:62449:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764745.297674:0:62449:0:(rw.c:1928:ll_readpage()) fast read pgno: 0 00000080:00200000:2.0:1775764745.297689:0:62449:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x9:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764745.297702:0:62449:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x9:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764745.297706:0:62449:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764745.297714:0:62449:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764745.297744:0:62449:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1775764745.297774:0:62449:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764745.297781:0:62449:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e243b0, osc lock ffff8b238a3f9300, flags 0x0 00010000:00010000:2.0:1775764745.297805:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1775764745.297826:0:62449:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1775764745.297842:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1775764745.297857:0:62449:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1775764745.297866:0:62449:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00000001:00010000:2.0:1775764745.297883:0:62449:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x9:0x0] 00000008:00010000:2.0:1775764745.297890:0:62449:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764745.297892:0:62449:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b2389e24530, osc lock ffff8b238a3f8850, flags 0x400001002 00010000:00010000:2.0:1775764745.297900:0:62449:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1775764745.297910:0:62449:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00000020:00200000:2.0:1775764745.297919:0:62449:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764745.297922:0:62449:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 7/0,2 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1775764745.297930:0:62449:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00000020:00200000:2.0:1775764745.297937:0:62449:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764745.297945:0:62449:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x9:0x0] updating i_size 15 i_blocks 11 00000020:00010000:2.0:1775764745.297949:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c248 (ffff8b23812ec248/0) at cl_lock_release():245 00000020:00010000:2.0:1775764745.297952:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c248 (ffff8b23812ec248/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764745.297955:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389e24530 (ffff8b23812ecc68/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764745.297959:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c248 (ffff8b23812ec248/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764745.297975:0:62449:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389e24530 (ffff8b23812ec248/1) at cl_lock_fini():72 00000008:00010000:2.0:1775764745.298026:0:62449:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b2383c9e480 00000008:00010000:2.0:1775764745.298075:0:46654:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b2383c9e480 00010000:00010000:2.0:1775764745.298099:0:46654:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 24 lru_type: 1 00000008:00100000:2.0:1775764745.298140:0:46654:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea5180 - object 0x240000400:282 offset 0<>4096 00000100:00100000:2.0:1775764745.298212:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764745.298231:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098617856:192.168.202.128@tcp:3:lfs.0 00000008:00010000:2.0:1775764745.304711:0:2379:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8b2383c9e480 00000020:00200000:2.0:1775764745.304728:0:2379:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1775764745.304764:0:2379:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8b2383c9e480 00000100:00100000:2.0:1775764745.304792:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098617856:192.168.202.128@tcp:3:lfs.0 00000080:00200000:2.0:1775764745.443653:0:62460:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764745.443690:0:62460:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764745.443774:0:62460:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 62460 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.443788:0:62460:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62460 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.443801:0:62460:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62460 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764745.443835:0:62460:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62460:1862022098618240:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764745.445326:0:62460:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1df00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764745.453290:0:62460:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62460:1862022098618240:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764745.453332:0:62460:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.453347:0:62460:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.453361:0:62460:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.453373:0:62460:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.453386:0:62460:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.453394:0:62460:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.453400:0:62460:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764745.453422:0:62460:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea5180 x1862022098618240/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764761 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764745.453435:0:62460:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.453482:0:62460:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764745.453491:0:62460:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764745.453532:0:62460:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764745.453541:0:62460:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be17 00000080:00010000:2.0:1775764745.453549:0:62460:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764745.453553:0:62460:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be17 from it ffffa60b01fd7bb0 00010000:00010000:2.0:1775764745.453556:0:62460:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.453568:0:62460:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764745.453602:0:62460:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764745.453623:0:62460:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764745.453696:0:62460:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffe3f1149b0 00000100:00100000:2.0:1775764745.453772:0:62460:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62460:1862022098618368:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764745.453865:0:62460:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1df00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764745.462546:0:62460:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62460:1862022098618368:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764745.462635:0:62460:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:1.0:1775764745.511520:0:62461:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764745.511558:0:62461:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764745.511581:0:62461:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764745.511601:0:62461:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764745.511610:0:62461:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764745.511620:0:62461:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764745.511626:0:62461:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be17 00000080:00010000:1.0:1775764745.511634:0:62461:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be17 from it ffffa60b0200fbf8 00010000:00010000:1.0:1775764745.511639:0:62461:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764745.511648:0:62461:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764745.511665:0:62461:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764745.511682:0:62461:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b23853c3380, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1775764745.511687:0:62461:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open 00000002:00010000:1.0:1775764745.511701:0:62461:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f23.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:1.0:1775764745.511756:0:62461:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17be1e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 62461 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764745.511764:0:62461:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17be1e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62461 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764745.511776:0:62461:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764745.511789:0:62461:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd180 x1862022098618624/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764745.511799:0:62461:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17be1e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62461 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764745.511821:0:62461:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62461:1862022098618624:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764745.511999:0:62461:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23894b6b40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764745.523727:0:62461:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62461:1862022098618624:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764745.523763:0:62461:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17be1e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62461 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.523779:0:62461:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17be1e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 62461 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.523787:0:62461:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17be1e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 62461 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.523798:0:62461:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17be1e lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 62461 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.523807:0:62461:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17be1e lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 62461 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.523812:0:62461:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8740000 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764745.523817:0:62461:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17be1e lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 62461 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.523825:0:62461:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17be1e lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 62461 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.523833:0:62461:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17be1e lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 62461 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.523840:0:62461:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764745.523848:0:62461:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17be1e lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 62461 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764745.523854:0:62461:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17be1e lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 62461 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764745.523873:0:62461:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff8b23864fd180 x1862022098618624/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/608 e 0 to 0 dl 1775764761 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764745.523889:0:62461:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1775764745.524059:0:62461:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764745.524078:0:62461:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764745.524093:0:62461:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764745.524106:0:62461:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764745.524114:0:62461:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764745.524120:0:62461:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764745.524124:0:62461:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be17 00000080:00010000:2.0:1775764745.524130:0:62461:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be17 from it ffffa60b0200fbb0 00010000:00010000:2.0:1775764745.524133:0:62461:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764745.524140:0:62461:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764745.524154:0:62461:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764745.524166:0:62461:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764745.524202:0:62461:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086915 arg=aec3d0 00000100:00100000:2.0:1775764745.524268:0:62461:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62461:1862022098618752:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764745.524361:0:62461:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384f1df00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764745.530978:0:62461:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:62461:1862022098618752:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764745.531231:0:62461:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:1.0:1775764745.602946:0:62468:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764745.602976:0:62468:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764745.602997:0:62468:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764745.603015:0:62468:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764745.603024:0:62468:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764745.603032:0:62468:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764745.603035:0:62468:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be17 00000080:00010000:1.0:1775764745.603042:0:62468:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be17 from it ffffa60b01f57b58 00010000:00010000:1.0:1775764745.603045:0:62468:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764745.603053:0:62468:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764745.603069:0:62468:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764745.603081:0:62468:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), flags=9 00000080:00200000:1.0:1775764745.603085:0:62468:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f23.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=getattr 00000002:00010000:1.0:1775764745.603097:0:62468:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f23.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764745.603141:0:62468:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17be25 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 62468 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764745.603149:0:62468:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17be25 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62468 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764745.603159:0:62468:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17be25 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62468 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764745.603185:0:62468:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job mmap_cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:62468:1862022098618880:192.168.202.128@tcp:101:mmap_cat.0 00000100:00100000:1.0:1775764745.603399:0:62468:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23894b6b40 going to sleep for 11 seconds 00010000:00010000:2.0:1775764746.170622:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00010000:1.0:1775764747.378345:0:2376:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 0000000093d2481a ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00000008:00010000:1.0:1775764747.378387:0:2376:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000093d2481a ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000001d779b53/0x73cfd86cfc17bd37 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03051c11 expref: -99 pid: 62071 timeout: 0 lvb_type: 1 lru_score: 29 lru_type: 1 00000100:00100000:1.0:1775764747.378436:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098619264:192.168.202.128@tcp:4:lfs.0 00010000:00010000:1.0:1775764749.240594:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764750.709305:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2378:1862022098619520:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764750.709344:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2378:1862022098619776:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764750.709366:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098620032:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764750.709379:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2378:1862022098619648:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764750.709389:0:2378:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 34359739179 gen 1 00000100:00100000:2.0:1775764750.709428:0:2378:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b2382dace00 x1862022098619136/t34359739179(34359739179) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764761 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'mmap_cat.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764750.709439:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098619904:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764750.709483:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098620160:192.168.202.128@tcp:400:kworker.0 00010000:00010000:1.0:1775764751.298956:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1775764752.038596:0:62890:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1775764752.038604:0:62890:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:2.0:1775764752.038621:0:62890:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764752.038687:0:62890:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 62890 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764752.038696:0:62890:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62890 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764752.038706:0:62890:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 62890 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764752.038732:0:62890:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job lctl:82781e18-e0e3-4f9f-9a2b-041bf0344414:62890:1862022098620288:192.168.202.128@tcp:101:lctl.0 00000100:00100000:2.0:1775764752.041569:0:62890:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b6543c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764752.046851:0:62890:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job lctl:82781e18-e0e3-4f9f-9a2b-041bf0344414:62890:1862022098620288:192.168.202.128@tcp:101:lctl.0 00010000:00010000:2.0:1775764752.046879:0:62890:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764752.046991:0:62890:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764752.047005:0:62890:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764752.047015:0:62890:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764752.047028:0:62890:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764752.047036:0:62890:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764752.047043:0:62890:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764752.047055:0:62890:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea4700 x1862022098620288/t0(0) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764768 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lctl.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764752.047068:0:62890:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764752.047080:0:62890:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2387c07800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764752.047086:0:62890:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764752.047115:0:62890:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764752.047122:0:62890:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17be33 00000080:00010000:2.0:1775764752.047127:0:62890:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:2.0:1775764752.047130:0:62890:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be33 from it ffffa60b00da3cb8 00010000:00010000:2.0:1775764752.047134:0:62890:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764752.047141:0:62890:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764752.047378:0:62890:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764752.047396:0:62890:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764752.047407:0:62890:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764752.047418:0:62890:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764752.047425:0:62890:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764752.047430:0:62890:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764752.047432:0:62890:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be17 00000080:00010000:2.0:1775764752.047437:0:62890:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be17 from it ffffa60b00da3bb0 00010000:00010000:2.0:1775764752.047439:0:62890:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764752.047445:0:62890:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764752.047514:0:62890:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764752.048608:0:62890:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200080:2.0:1775764752.048642:0:62890:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=8080667f arg=7ffd5260ded0 00000080:00200008:2.0:1775764752.048660:0:62890:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000100:00100000:2.0:1775764752.265052:0:62303:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:62303:1862022098620416:192.168.202.128@tcp:60:lhsmtool_posix.0 00000080:00200008:2.0:1775764752.265124:0:62303:0:(file.c:447:ll_file_release()) START file fid:[0x200000002:0x2:0x0](ffff8b2390ec5348), flags 100000 00000002:00100000:2.0:1775764752.265185:0:62303:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fca80 x1862022098603392/t34359739144(34359739144) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/4688 e 0 to 0 dl 1775764757 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764752.265196:0:62303:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764752.265200:0:62303:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4700 x1862022098620672/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764752.265220:0:62303:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:62303:1862022098620672:192.168.202.128@tcp:35:lhsmtool_posix.0 00000100:00100000:2.0:1775764752.265419:0:62303:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b6543c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764752.270715:0:62303:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:62303:1862022098620672:192.168.202.128@tcp:35:lhsmtool_posix.0 00000002:00100000:2.0:1775764752.270746:0:62303:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fca80 x1862022098603392/t34359739144(34359739144) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/4688 e 0 to 0 dl 1775764757 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1775764752.270806:0:62303:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 100000 00010000:00010000:2.0:1775764752.317475:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1775764752.752000:0:62908:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764752.752015:0:62908:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:2.0:1775764752.752044:0:62908:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764752.752065:0:62908:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764752.752082:0:62908:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764752.752090:0:62908:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764752.752098:0:62908:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:2.0:1775764752.752102:0:62908:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17be33 00000080:00010000:2.0:1775764752.752109:0:62908:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be33 from it ffffa60b00d17cb8 00010000:00010000:2.0:1775764752.752113:0:62908:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764752.752122:0:62908:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200008:2.0:1775764752.760517:0:62908:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764752.760531:0:62908:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764752.760569:0:62908:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764752.760590:0:62908:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764752.760605:0:62908:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1775764752.760612:0:62908:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764752.760621:0:62908:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764752.760625:0:62908:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be17 00000080:00010000:2.0:1775764752.760632:0:62908:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be17 from it ffffa60b00d17cb8 00010000:00010000:2.0:1775764752.760635:0:62908:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764752.760643:0:62908:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764753.337087:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764755.514613:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2379:1862022098620800:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764755.514679:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098621184:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764755.514706:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098621440:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764755.514764:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098621056:192.168.202.128@tcp:9:kworker.0 00000100:00100000:2.0:1775764755.524851:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2379:1862022098620800:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764755.524876:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098621440:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764755.526301:0:2379:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 34359739180 gen 1 00000100:00100000:2.0:1775764755.526322:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098621184:192.168.202.128@tcp:400:kworker.0 00000100:00080000:2.0:1775764755.550921:0:2375:0:(import.c:2005:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff8b23864f1f80 x1862022098621056/t0(0) o9->lustre-OST0001-osc-ffff8b238a3bf000@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764765 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764755.550936:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b5000 lustre-OST0001_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:2.0:1775764755.550940:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098621056:192.168.202.128@tcp:9:kworker.0 00000001:02000400:1.0:1775764758.500930:0:63067:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-pcc test 24a: Test RO-PCC with non-root user === 15:59:16 (1775764756) 00000080:00200008:2.0:1775764760.878531:0:63167:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1775764760.878548:0:63167:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764760.878571:0:63167:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764760.878593:0:63167:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1775764760.878609:0:63167:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1775764760.878618:0:63167:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764760.878626:0:63167:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764760.878629:0:63167:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be17 00000080:00010000:2.0:1775764760.878636:0:63167:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be17 from it ffffa60b01e03cb8 00010000:00010000:2.0:1775764760.878641:0:63167:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1775764760.878650:0:63167:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200008:2.0:1775764760.879149:0:63167:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1775764760.879154:0:63167:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:2.0:1775764760.879165:0:63167:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764760.879176:0:63167:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764760.879184:0:63167:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764760.879190:0:63167:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764760.879194:0:63167:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:2.0:1775764760.879196:0:63167:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17be33 00000080:00010000:2.0:1775764760.879200:0:63167:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be33 from it ffffa60b01e03cb8 00010000:00010000:2.0:1775764760.879202:0:63167:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764760.879209:0:63167:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23ac6eac00/0x73cfd86cfc17be33 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051ffa expref: -99 pid: 62890 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764760.879316:0:63167:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764760.879324:0:63167:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764760.879331:0:63167:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1775764760.879338:0:63167:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1775764760.879344:0:63167:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764760.879347:0:63167:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764760.879349:0:63167:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be17 00000080:00010000:2.0:1775764760.879352:0:63167:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be17 from it ffffa60b01e03bb0 00010000:00010000:2.0:1775764760.879354:0:63167:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1775764760.879361:0:63167:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:2.0:1775764760.879368:0:63167:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764760.879373:0:63167:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 100000 00000080:00200000:1.0:1775764760.881968:0:63168:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764760.881990:0:63168:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764760.882005:0:63168:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764760.882019:0:63168:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1775764760.882027:0:63168:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764760.882034:0:63168:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764760.882037:0:63168:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be17 00000080:00010000:1.0:1775764760.882044:0:63168:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be17 from it ffffa60b01e2bbb0 00010000:00010000:1.0:1775764760.882047:0:63168:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764760.882061:0:63168:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764760.882074:0:63168:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764760.882081:0:63168:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 100000 00000080:00200000:1.0:1775764760.882100:0:63168:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764760.882105:0:63168:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764760.882111:0:63168:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764760.882118:0:63168:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764760.882124:0:63168:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764760.882127:0:63168:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764760.882129:0:63168:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be17 00000080:00010000:1.0:1775764760.882132:0:63168:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be17 from it ffffa60b01e2bbf8 00010000:00010000:1.0:1775764760.882134:0:63168:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764760.882156:0:63168:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1775764760.882165:0:63168:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764760.882171:0:63168:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=.lustre, flags=17 00000080:00200000:1.0:1775764760.882176:0:63168:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000002:0x1:0x0](ffff8b2389909988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764760.882187:0:63168:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=fid, dir=[0x200000002:0x1:0x0](ffff8b2389909988), file ffff8b238629f200, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1775764760.882194:0:63168:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=fid, dir=[0x200000002:0x1:0x0](ffff8b2389909988), intent=open 00000002:00010000:1.0:1775764760.882209:0:63168:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: fid,[0x0:0x0:0x0]) in obj [0x200000002:0x1:0x0], intent: open flags 0100001 00010000:00010000:1.0:1775764760.882277:0:63168:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17be3a lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 63168 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764760.882285:0:63168:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17be3a lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63168 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764760.882295:0:63168:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764760.882307:0:63168:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fce00 x1862022098622336/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764760.882316:0:63168:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17be3a lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63168 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764760.882338:0:63168:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:63168:1862022098622336:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764760.882544:0:63168:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238a33a9c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764760.920558:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098622592:192.168.202.128@tcp:103:ldlm_bl.0 00000080:00200000:1.0:1775764761.081405:0:63176:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764761.081436:0:63176:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764761.081722:0:63176:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764761.081743:0:63176:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1775764761.081753:0:63176:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764761.081762:0:63176:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764761.081765:0:63176:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be17 00000080:00010000:1.0:1775764761.081772:0:63176:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be17 from it ffffa60b01e4bbb0 00010000:00010000:1.0:1775764761.081776:0:63176:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764761.081785:0:63176:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:1.0:1775764761.081801:0:63176:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764761.081809:0:63176:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200080:2.0:1775764761.082965:0:63176:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=8080667f arg=7ffe29835890 00000080:00200008:2.0:1775764761.082990:0:63176:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200000:1.0:1775764761.147987:0:63177:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764761.148014:0:63177:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764761.148032:0:63177:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764761.148048:0:63177:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:1.0:1775764761.148055:0:63177:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764761.148064:0:63177:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764761.148067:0:63177:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be17 00000080:00010000:1.0:1775764761.148074:0:63177:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be17 from it ffffa60b01e3bbb0 00010000:00010000:1.0:1775764761.148078:0:63177:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764761.148088:0:63177:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1775764761.148103:0:63177:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764761.148111:0:63177:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200080:1.0:1775764761.148137:0:63177:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=8080667f arg=7ffd95be2830 00000080:00200008:1.0:1775764761.148151:0:63177:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00010000:00010000:1.0:1775764761.209279:0:63178:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:1.0:1775764761.209342:0:63178:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:1.0:1775764761.209349:0:63178:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764761.209352:0:63178:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb200/0x73cfd86cfc17be17 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051f7c expref: -99 pid: 62460 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000020:00100000:1.0:1775764761.209387:0:63178:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1775764761.209402:0:63178:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ffb80 x1862022098622848/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 560/984 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764761.209431:0:63178:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job mkdir:fcf15866-4739-4595-8fe1-44abde7ee9d7:63178:1862022098622848:192.168.202.128@tcp:36:mkdir.0 00000100:00100000:1.0:1775764761.209573:0:63178:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238a33ab40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764761.239137:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:2.0:1775764761.253008:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2378:1862022098622976:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764761.269423:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2378:1862022098622976:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764761.277926:0:63178:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job mkdir:fcf15866-4739-4595-8fe1-44abde7ee9d7:63178:1862022098622848:192.168.202.128@tcp:36:mkdir.0 00000080:00200000:2.0:1775764761.277981:0:63178:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8) 00000080:00200000:2.0:1775764761.277989:0:63178:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xb:0x0], updating i_size 512 00000080:00200000:2.0:1775764761.298039:0:63179:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764761.298072:0:63179:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764761.298140:0:63179:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 63179 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.298390:0:63179:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63179 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.298404:0:63179:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63179 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764761.298435:0:63179:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job chmod:fcf15866-4739-4595-8fe1-44abde7ee9d7:63179:1862022098623104:192.168.202.128@tcp:101:chmod.0 00000100:00100000:2.0:1775764761.298772:0:63179:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0e2d780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764761.307975:0:63179:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job chmod:fcf15866-4739-4595-8fe1-44abde7ee9d7:63179:1862022098623104:192.168.202.128@tcp:101:chmod.0 00010000:00010000:2.0:1775764761.308002:0:63179:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.308016:0:63179:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.308027:0:63179:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.308036:0:63179:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.308048:0:63179:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.308055:0:63179:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.308062:0:63179:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764761.308072:0:63179:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea4700 x1862022098623104/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764777 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'chmod.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764761.308085:0:63179:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.308098:0:63179:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764761.308104:0:63179:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764761.308130:0:63179:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764761.308137:0:63179:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:2.0:1775764761.308152:0:63179:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764761.308155:0:63179:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b014afb58 00010000:00010000:2.0:1775764761.308158:0:63179:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.308167:0:63179:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764761.308197:0:63179:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764761.308221:0:63179:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=d24a.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), flags=9 00000080:00200000:2.0:1775764761.308225:0:63179:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=d24a.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=getattr 00000002:00010000:2.0:1775764761.308255:0:63179:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d24a.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764761.308301:0:63179:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 63179 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.308308:0:63179:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63179 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.308317:0:63179:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63179 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764761.308339:0:63179:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job chmod:fcf15866-4739-4595-8fe1-44abde7ee9d7:63179:1862022098623232:192.168.202.128@tcp:101:chmod.0 00000100:00100000:2.0:1775764761.308441:0:63179:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0e2d780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764761.318956:0:63179:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job chmod:fcf15866-4739-4595-8fe1-44abde7ee9d7:63179:1862022098623232:192.168.202.128@tcp:101:chmod.0 00010000:00010000:2.0:1775764761.318983:0:63179:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.319008:0:63179:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.319016:0:63179:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.319027:0:63179:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.319038:0:63179:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.319051:0:63179:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.319060:0:63179:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.319067:0:63179:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764761.319078:0:63179:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea4700 x1862022098623232/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764777 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'chmod.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764761.319092:0:63179:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.319102:0:63179:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/11 (0 0) 00000080:00200000:2.0:1775764761.319108:0:63179:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764761.319133:0:63179:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xb:0x0], updating i_size 512 00000080:00200000:2.0:1775764761.319137:0:63179:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0xb:0x0](ffff8b2390ec00c8): rc = 0 00000080:00010000:2.0:1775764761.319151:0:63179:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xb:0x0](ffff8b2390ec00c8) for lock 0x73cfd86cfc17be48 00000080:00010000:2.0:1775764761.319165:0:63179:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xb:0x0](ffff8b2390ec00c8) 00000080:00010000:2.0:1775764761.319169:0:63179:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be48 from it ffffa60b014afb20 00010000:00010000:2.0:1775764761.319172:0:63179:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.319181:0:63179:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1775764761.319240:0:63179:0:(file.c:6226:ll_getattr_dentry()) START file d24a.sanity-pcc:[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764761.319248:0:63179:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8),name=d24a.sanity-pcc 00000002:00010000:2.0:1775764761.319279:0:63179:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xb:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764761.319289:0:63179:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764761.319299:0:63179:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764761.319304:0:63179:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764761.319309:0:63179:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xb:0x0](ffff8b2390ec00c8) 00000080:00010000:2.0:1775764761.319311:0:63179:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xb:0x0](ffff8b2390ec00c8) for lock 0x73cfd86cfc17be48 00000080:00010000:2.0:1775764761.319314:0:63179:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be48 from it ffffa60b014afcb8 00010000:00010000:2.0:1775764761.319317:0:63179:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764761.319324:0:63179:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764761.319377:0:63179:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764761.319385:0:63179:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764761.319392:0:63179:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764761.319400:0:63179:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764761.319405:0:63179:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764761.319409:0:63179:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764761.319411:0:63179:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:2.0:1775764761.319416:0:63179:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b014afc08 00010000:00010000:2.0:1775764761.319418:0:63179:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764761.319424:0:63179:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764761.319432:0:63179:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764761.319438:0:63179:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24a.sanity-pcc, flags=9 00000080:00200008:2.0:1775764761.319449:0:63179:0:(llite_lib.c:2274:ll_setattr_raw()) START file d24a.sanity-pcc:[0x200003ab4:0xb:0x0](ffff8b2390ec00c8) current size 512, valid attrs 41, mode 41ff, uid -1604198400, gid -29917, new size 0, atime 1775764761.000000000, mtime 1775764761.000000000, ctime 1775764761.000000000 00010000:00010000:2.0:1775764761.319515:0:63179:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764761.319559:0:63179:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764761.319565:0:63179:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764761.319567:0:63179:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be48 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052055 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000020:00100000:2.0:1775764761.319587:0:63179:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764761.319593:0:63179:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4700 x1862022098623488/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764761.319614:0:63179:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job chmod:fcf15866-4739-4595-8fe1-44abde7ee9d7:63179:1862022098623488:192.168.202.128@tcp:36:chmod.0 00000100:00100000:2.0:1775764761.320368:0:63179:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0e2d780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764761.337194:0:63179:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job chmod:fcf15866-4739-4595-8fe1-44abde7ee9d7:63179:1862022098623488:192.168.202.128@tcp:36:chmod.0 00000080:00200000:2.0:1775764761.337240:0:63179:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xb:0x0], updating i_size 512 00000080:00200008:2.0:1775764761.337252:0:63179:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file d24a.sanity-pcc:[0x200003ab4:0xb:0x0](ffff8b2390ec00c8) current size 512, valid attrs 41, mode 41ff, uid -1604198400, gid -29917, new size 0, atime 1775764761.000000000, mtime 1775764761.000000000, ctime 1775764761.000000000, rc 0 00000080:00200000:1.0:1775764761.502070:0:63185:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764761.502100:0:63185:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764761.502121:0:63185:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764761.502138:0:63185:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764761.502146:0:63185:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:1.0:1775764761.502156:0:63185:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764761.502160:0:63185:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:1.0:1775764761.502168:0:63185:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01db3bf8 00010000:00010000:1.0:1775764761.502172:0:63185:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764761.502182:0:63185:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764761.502201:0:63185:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000100:00100000:1.0:1775764761.502281:0:63185:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:63185:1862022098623616:192.168.202.128@tcp:49:dd.500 00000100:00100000:1.0:1775764761.502393:0:63185:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238a33a000 going to sleep for 11 seconds 00010000:00010000:1.0:1775764761.531206:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1775764761.546078:0:63185:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:2.0:1775764761.546135:0:63185:0:(osc_cache.c:848:osc_extent_find()) ### extent: 0000000051656cc4 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764761.546168:0:63185:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b23899c2bb8, count 1, dirtied 1 00000080:00200000:2.0:1775764761.546175:0:63185:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:2.0:1775764761.546178:0:63185:0:(vvp_io.c:1353:vvp_io_write_start()) f24a.sanity-pcc: write bytes 0, result: 1024 00000020:00010000:2.0:1775764761.546200:0:63185:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2389eade48 (ffff8b2383fb5908/0) at cl_lock_release():245 00000020:00010000:2.0:1775764761.546203:0:63185:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389eade48 (ffff8b2383fb5908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764761.546206:0:63185:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384986030 (ffff8b2383fb5248/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764761.546213:0:63185:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.546240:0:63185:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1775764761.546269:0:63185:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389eade48 (ffff8b2383fb5908/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764761.546273:0:63185:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384986030 (ffff8b2383fb5908/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764761.546286:0:63185:0:(vvp_io.c:517:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:2.0:1775764761.546293:0:63185:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:2.0:1775764761.546336:0:63185:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xc:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764761.546339:0:63185:0:(file.c:2206:ll_file_io_generic()) f24a.sanity-pcc: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:2.0:1775764761.546349:0:63185:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0xc:0x0] updating i_size 1024 i_blocks 1 00000080:00200000:2.0:1775764761.546354:0:63185:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:2.0:1775764761.546409:0:63185:0:(file.c:447:ll_file_release()) START file f24a.sanity-pcc:[0x200003ab4:0xc:0x0](ffff8b23899c2a08), flags 100001 00000002:00100000:2.0:1775764761.546568:0:63185:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b2382dadf80 x1862022098624128/t34359739184(34359739184) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764777 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.500' uid:500 gid:500 projid:0 00000020:00100000:2.0:1775764761.546581:0:63185:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764761.546586:0:63185:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6680 x1862022098624512/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764761.546610:0:63185:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:63185:1862022098624512:192.168.202.128@tcp:35:dd.500 00000100:00100000:2.0:1775764761.546709:0:63185:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0e2d780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764761.556817:0:63185:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:63185:1862022098624512:192.168.202.128@tcp:35:dd.500 00000002:00100000:2.0:1775764761.556856:0:63185:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b2382dadf80 x1862022098624128/t34359739184(34359739184) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764777 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.500' uid:500 gid:500 projid:0 00010000:00010000:2.0:1775764761.691393:0:63191:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764761.691413:0:63191:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:2.0:1775764761.691425:0:63191:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764761.691429:0:63191:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:2.0:1775764761.691440:0:63191:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01e8bcb8 00010000:00010000:2.0:1775764761.691444:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764761.691793:0:63191:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764761.691961:0:63191:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764761.691979:0:63191:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764761.691990:0:63191:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764761.692001:0:63191:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764761.692007:0:63191:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:2.0:1775764761.692012:0:63191:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764761.692015:0:63191:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:2.0:1775764761.692018:0:63191:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01e8bb58 00010000:00010000:2.0:1775764761.692021:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764761.692028:0:63191:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764761.692037:0:63191:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764761.692044:0:63191:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24a.sanity-pcc, flags=8 00000080:00200008:2.0:1775764761.692053:0:63191:0:(file.c:6226:ll_getattr_dentry()) START file d24a.sanity-pcc:[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764761.692057:0:63191:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8),name=d24a.sanity-pcc 00000002:00010000:2.0:1775764761.692068:0:63191:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xb:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764761.692074:0:63191:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 30 res 8589949620/11 (0 0) 00010000:00010000:2.0:1775764761.692115:0:63191:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 63191 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.692122:0:63191:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63191 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.692132:0:63191:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63191 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764761.692163:0:63191:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63191:1862022098624640:192.168.202.128@tcp:101:lfs.500 00000100:00100000:2.0:1775764761.692957:0:63191:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0e2d780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764761.705308:0:63191:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63191:1862022098624640:192.168.202.128@tcp:101:lfs.500 00010000:00010000:2.0:1775764761.705339:0:63191:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.705358:0:63191:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.705367:0:63191:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.705379:0:63191:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.705390:0:63191:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.705404:0:63191:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.705412:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.705419:0:63191:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764761.705437:0:63191:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea4a80 x1862022098624640/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764777 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.500' uid:500 gid:500 projid:0 00000002:00010000:2.0:1775764761.705450:0:63191:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.705719:0:63191:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/11 (0 0) 00000080:00200000:2.0:1775764761.705725:0:63191:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 0 00000080:00200000:2.0:1775764761.705744:0:63191:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xb:0x0], updating i_size 512 00000080:00010000:2.0:1775764761.705749:0:63191:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xb:0x0](ffff8b2390ec00c8) for lock 0x73cfd86cfc17be64 00000080:00010000:2.0:1775764761.705757:0:63191:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xb:0x0](ffff8b2390ec00c8) 00000080:00010000:2.0:1775764761.705760:0:63191:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be64 from it ffffa60b01e8bcb8 00010000:00010000:2.0:1775764761.705764:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.705772:0:63191:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764761.705844:0:63191:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764761.705859:0:63191:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764761.705868:0:63191:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764761.705879:0:63191:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764761.705886:0:63191:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:2.0:1775764761.705890:0:63191:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764761.705893:0:63191:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:2.0:1775764761.705897:0:63191:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01e8bb58 00010000:00010000:2.0:1775764761.705900:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764761.705907:0:63191:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764761.705915:0:63191:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764761.705922:0:63191:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24a.sanity-pcc, flags=24 00000080:00200000:2.0:1775764761.705927:0:63191:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), inode mode 41ff mask 1 00000100:00100000:2.0:1775764761.705973:0:63191:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63191:1862022098624768:192.168.202.128@tcp:49:lfs.500 00000100:00100000:2.0:1775764761.706060:0:63191:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0e2d780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764761.721562:0:63191:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63191:1862022098624768:192.168.202.128@tcp:49:lfs.500 00000080:00200000:2.0:1775764761.721601:0:63191:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f24a.sanity-pcc, flags=8 00000080:00200008:2.0:1775764761.721616:0:63191:0:(file.c:6226:ll_getattr_dentry()) START file f24a.sanity-pcc:[0x200003ab4:0xc:0x0](ffff8b23899c2a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764761.721623:0:63191:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0xc:0x0](ffff8b23899c2a08),name=f24a.sanity-pcc 00000002:00010000:2.0:1775764761.721646:0:63191:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xc:0x0]) in obj [0x200003ab4:0xb:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764761.721665:0:63191:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17be56 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052094 expref: -99 pid: 63185 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764761.721679:0:63191:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17be56 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052094 expref: -99 pid: 63185 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764761.721819:0:63191:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200003ab4:0xb:0x0] suppgids -1 500: rc 1 00000080:00010000:2.0:1775764761.721827:0:63191:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xc:0x0](ffff8b23899c2a08) 00000080:00010000:2.0:1775764761.721831:0:63191:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xc:0x0](ffff8b23899c2a08) for lock 0x73cfd86cfc17be56 00000080:00010000:2.0:1775764761.721972:0:63191:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be56 from it ffffa60b01e8bcb8 00010000:00010000:2.0:1775764761.721979:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17be56 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052094 expref: -99 pid: 63185 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764761.721989:0:63191:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17be56 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052094 expref: -99 pid: 63185 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764761.722074:0:63191:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1775764761.722084:0:63191:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0xc:0x0] 00000008:00010000:2.0:1775764761.722114:0:63191:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764761.722118:0:63191:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8db0, osc lock ffff8b238a3f8850, flags 0x400001002 00010000:00010000:2.0:1775764761.722136:0:63191:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764761.722148:0:63191:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1775764761.722159:0:63191:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764761.722162:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764761.722169:0:63191:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1775764761.722180:0:63191:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764761.722189:0:63191:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0xc:0x0] updating i_size 1024 i_blocks 1 00000020:00010000:2.0:1775764761.722193:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027fd98 (ffff8b23812ec878/0) at cl_lock_release():245 00000020:00010000:2.0:1775764761.722196:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027fd98 (ffff8b23812ec878/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764761.722198:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8db0 (ffff8b23812ec638/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764761.722202:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027fd98 (ffff8b23812ec878/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764761.722204:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8db0 (ffff8b23812ec878/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764761.722222:0:63191:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xc:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764761.722414:0:63191:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764761.722424:0:63191:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764761.722433:0:63191:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764761.722442:0:63191:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764761.722448:0:63191:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:2.0:1775764761.722478:0:63191:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764761.722481:0:63191:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:2.0:1775764761.722485:0:63191:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01e8bbf8 00010000:00010000:2.0:1775764761.722488:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764761.722496:0:63191:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764761.722505:0:63191:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764761.722510:0:63191:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24a.sanity-pcc, flags=25 00000080:00200000:2.0:1775764761.722515:0:63191:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), inode mode 41ff mask 1 00000080:00200000:2.0:1775764761.722518:0:63191:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f24a.sanity-pcc, flags=265 00000080:00200000:2.0:1775764761.722522:0:63191:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xc:0x0](ffff8b23899c2a08), inode mode 81a4 mask 44 00000080:00200008:2.0:1775764761.722529:0:63191:0:(file.c:1057:ll_file_open()) START file f24a.sanity-pcc:[0x200003ab4:0xc:0x0](ffff8b23899c2a08), flags 100000 00000002:00010000:2.0:1775764761.722541:0:63191:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xc:0x0]) in obj [0x200003ab4:0xb:0x0], intent: open flags 0240100001 00010000:00010000:2.0:1775764761.722585:0:63191:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17be6b lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 63191 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.722593:0:63191:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17be6b lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 63191 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764761.722604:0:63191:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764761.722613:0:63191:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4a80 x1862022098625024/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764761.722623:0:63191:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17be6b lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 63191 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764761.722643:0:63191:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63191:1862022098625024:192.168.202.128@tcp:101:lfs.500 00000100:00100000:2.0:1775764761.723247:0:63191:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0e2d780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764761.739004:0:63191:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63191:1862022098625024:192.168.202.128@tcp:101:lfs.500 00010000:00010000:2.0:1775764761.739033:0:63191:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17be6b lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520be expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.739049:0:63191:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17be6b lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520be expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.739056:0:63191:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17be6b lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520be expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.739067:0:63191:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17be6b lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030520be expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.739083:0:63191:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17be6b lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520be expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.739095:0:63191:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17be6b lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030520be expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.739104:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17be6b lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030520be expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.739111:0:63191:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17be6b lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030520be expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764761.739121:0:63191:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23b9ea4a80 x1862022098625024/t34359739186(34359739186) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764777 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.500' uid:500 gid:500 projid:0 00000002:00100000:2.0:1775764761.739134:0:63191:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4a80 x1862022098625024/t34359739186(34359739186) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764777 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.500' uid:500 gid:500 projid:0 00000002:00010000:2.0:1775764761.739145:0:63191:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17be6b lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520be expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764761.739158:0:63191:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17be6b lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520be expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.739169:0:63191:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17be56 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052094 expref: -99 pid: 63185 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764761.739180:0:63191:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17be56 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052094 expref: -99 pid: 63185 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764761.739188:0:63191:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17be6b lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520be expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.739194:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17be6b lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e030520be expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.739201:0:63191:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17be6b lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e030520be expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.739236:0:63191:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:2.0:1775764761.739244:0:63191:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab4:0xb:0x0] suppgids -1 500: rc 0 00000080:00200000:2.0:1775764761.739363:0:63191:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0xc:0x0]: layout version change: 0 -> 0 00000080:00010000:2.0:1775764761.739375:0:63191:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xc:0x0](ffff8b23899c2a08) for lock 0x73cfd86cfc17be56 00000080:00010000:2.0:1775764761.739382:0:63191:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be56 from it ffffa60b01e8bbf0 00010000:00010000:2.0:1775764761.739385:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17be56 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052094 expref: -99 pid: 63185 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764761.739394:0:63191:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17be56 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052094 expref: -99 pid: 63185 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000002:00100000:2.0:1775764761.739418:0:63191:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4a80 x1862022098625024/t34359739186(34359739186) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764777 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.500' uid:500 gid:500 projid:0 00000080:00200080:2.0:1775764761.740067:0:63191:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0xc:0x0](0000000000000000) cmd=400866fc arg=7fff4aff9dc8 00000080:00010000:2.0:1775764761.740110:0:63191:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0xc:0x0](ffff8b23899c2a08) 00000002:00010000:2.0:1775764761.740122:0:63191:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xc:0x0]) in obj [0x200003ab4:0xc:0x0], intent: layout flags 02 00010000:00010000:2.0:1775764761.740135:0:63191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17be56 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052094 expref: -99 pid: 63185 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764761.740153:0:63191:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0xc:0x0]: losing layout lock 00010000:00010000:2.0:1775764761.740182:0:63191:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17be56 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052094 expref: -99 pid: 63185 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764761.740189:0:63191:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764761.740191:0:63191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044800/0x73cfd86cfc17be56 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052094 expref: -99 pid: 63185 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764761.740217:0:63191:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0xc:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 63191 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.740224:0:63191:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0xc:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63191 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764761.740241:0:63191:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764761.740248:0:63191:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5180 x1862022098625408/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/880 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764761.740269:0:63191:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0xc:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63191 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764761.740285:0:63191:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63191:1862022098625408:192.168.202.128@tcp:101:lfs.500 00000100:00100000:2.0:1775764761.742010:0:63191:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0e2d0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764761.758182:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:2.0:1775764761.760943:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098625152:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764761.783367:0:63191:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63191:1862022098625408:192.168.202.128@tcp:101:lfs.500 00010000:00010000:2.0:1775764761.783392:0:63191:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 4/0,1 mode: --/EX res: [0x200003ab4:0xc:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.783410:0:63191:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 4/0,1 mode: --/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.783422:0:63191:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.783432:0:63191:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.783445:0:63191:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.783506:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.783514:0:63191:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764761.783526:0:63191:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea5180 x1862022098625408/t34359739187(34359739187) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/600 e 0 to 0 dl 1775764777 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'lfs.500' uid:500 gid:500 projid:0 00000002:00010000:2.0:1775764761.783545:0:63191:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764761.783557:0:63191:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.783574:0:63191:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/12 (0 0) 00000080:00200000:2.0:1775764761.783580:0:63191:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0xc:0x0] suppgids 500 500: rc 0 00000080:00010000:2.0:1775764761.783589:0:63191:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xc:0x0](ffff8b23899c2a08) for lock 0x73cfd86cfc17be72 00000080:00010000:2.0:1775764761.783606:0:63191:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be72 from it ffffa60b01e8bc10 00010000:00010000:2.0:1775764761.783610:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764761.783617:0:63191:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764761.783639:0:63191:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764761.783649:0:63191:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764761.783656:0:63191:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0xc:0x0](00000000ca5122fa) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764761.783745:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764761.783752:0:63191:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764761.783779:0:63191:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764761.783790:0:63191:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:2.0:1775764761.783828:0:63191:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 0000000051656cc4 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:2.0:1775764761.783853:0:63191:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 0000000051656cc4 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764761.784309:0:63191:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:2.0:1775764761.784390:0:63191:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea6d80 - object 0x280000400:282 offset 0<>1024 00000100:00100000:2.0:1775764761.784449:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764761.784490:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098625536:192.168.202.128@tcp:4:dd.500 00000008:00010000:2.0:1775764761.801211:0:2379:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 0000000051656cc4 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:2.0:1775764761.801291:0:2379:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000051656cc4 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:2.0:1775764761.801322:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098625536:192.168.202.128@tcp:4:dd.500 00000100:00100000:1.0:1775764761.813740:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098625664:192.168.202.128@tcp:3:lfs.0 00000020:00010000:2.0:1775764761.821535:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2389ead998 (ffff8b2383fb5758/0) at cl_lock_release():245 00000020:00010000:2.0:1775764761.821541:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389ead998 (ffff8b2383fb5758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764761.821545:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384986930 (ffff8b2383fb5008/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764761.821556:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764761.821573:0:63191:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00000020:00010000:2.0:1775764761.821589:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389ead998 (ffff8b2383fb5758/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764761.821592:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384986930 (ffff8b2383fb5758/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764761.821600:0:63191:0:(vvp_io.c:517:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:2.0:1775764761.821622:0:63191:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xc:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764761.821627:0:63191:0:(file.c:2206:ll_file_io_generic()) f24a.sanity-pcc: 1 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:2.0:1775764761.821632:0:63191:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 1024 00000080:00200008:2.0:1775764761.821780:0:63191:0:(file.c:2454:do_file_read_iter()) START file f24a.sanity-pcc:[0x200003ab4:0xc:0x0], ppos: 1024, count: 1048576 00000080:00200000:2.0:1775764761.821806:0:63191:0:(file.c:2058:ll_file_io_generic()) f24a.sanity-pcc: read ppos: 1024, bytes: 1048576 00000080:00200000:2.0:1775764761.821811:0:63191:0:(file.c:1815:ll_io_set_mirror()) f24a.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764761.821818:0:63191:0:(cl_io.c:214:cl_io_rw_init()) header@0000000042a321a3[0x110, 2, [0x200003ab4:0xc:0x0] hash] 00000020:00200000:2.0:1775764761.821820:0:63191:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [1024, 1049600) 0 0 00000080:00200000:2.0:1775764761.821824:0:63191:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764761.821850:0:63191:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0xc:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764761.821854:0:63191:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764761.821863:0:63191:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 1024 chunk: [1024, 1049600) 1049600, 1048576 00020000:00200000:2.0:1775764761.821868:0:63191:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764761.821909:0:63191:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 256] 00000008:00010000:2.0:1775764761.821924:0:63191:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764761.821929:0:63191:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8930, osc lock ffff8b238a3f8850, flags 0x400000000000000 00010000:00010000:2.0:1775764761.821951:0:63191:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00010000:00010000:2.0:1775764761.821963:0:63191:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1052671) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00010000:00010000:2.0:1775764761.821974:0:63191:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00010000:00010000:2.0:1775764761.821988:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00010000:00010000:2.0:1775764761.821994:0:63191:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00000080:00200000:2.0:1775764761.822003:0:63191:0:(vvp_io.c:826:vvp_io_read_start()) f24a.sanity-pcc: read [1024, 1049600) 00000001:00010000:2.0:1775764761.822019:0:63191:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0xc:0x0] 00000008:00010000:2.0:1775764761.822024:0:63191:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764761.822026:0:63191:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8830, osc lock ffff8b238a3f8000, flags 0x400000400001002 00010000:00010000:2.0:1775764761.822033:0:63191:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00010000:00010000:2.0:1775764761.822042:0:63191:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00000020:00200000:2.0:1775764761.822050:0:63191:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764761.822053:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00010000:00010000:2.0:1775764761.822060:0:63191:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 6/1,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00000020:00200000:2.0:1775764761.822066:0:63191:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764761.822073:0:63191:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0xc:0x0] updating i_size 1024 i_blocks 1 00000020:00010000:2.0:1775764761.822076:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764761.822079:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764761.822082:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8830 (ffff8b23812ec5a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764761.822084:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764761.822086:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8830 (ffff8b23812eca28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764761.822092:0:63191:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:1024 00000020:00010000:2.0:1775764761.822097:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764761.822099:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764761.822101:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8930 (ffff8b23812ec5a8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764761.822104:0:63191:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00010000:00010000:2.0:1775764761.822111:0:63191:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000020:00010000:2.0:1775764761.822119:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764761.822122:0:63191:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8930 (ffff8b23812eca28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764761.822125:0:63191:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1775764761.822158:0:63191:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xc:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764761.822161:0:63191:0:(file.c:2206:ll_file_io_generic()) f24a.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1775764761.822164:0:63191:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:2.0:1775764761.822372:0:63191:0:(file.c:447:ll_file_release()) START file f24a.sanity-pcc:[0x200003ab4:0xc:0x0](ffff8b23899c2a08), flags 100000 00000002:00100000:2.0:1775764761.822431:0:63191:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea4a80 x1862022098625024/t34359739186(34359739186) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764777 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.500' uid:500 gid:500 projid:0 00000020:00100000:2.0:1775764761.822443:0:63191:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764761.822449:0:63191:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7480 x1862022098625920/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764761.822514:0:63191:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63191:1862022098625920:192.168.202.128@tcp:35:lfs.500 00000100:00100000:2.0:1775764761.822593:0:63191:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0e2d0c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764761.837342:0:63191:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63191:1862022098625920:192.168.202.128@tcp:35:lfs.500 00000002:00100000:2.0:1775764761.837392:0:63191:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea4a80 x1862022098625024/t34359739186(34359739186) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764777 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.500' uid:500 gid:500 projid:0 00000080:00200000:1.0:1775764762.008353:0:63201:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764762.008382:0:63201:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764762.008400:0:63201:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764762.008415:0:63201:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764762.008424:0:63201:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764762.008432:0:63201:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764762.008436:0:63201:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:1.0:1775764762.008443:0:63201:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01e4bbf8 00010000:00010000:1.0:1775764762.008447:0:63201:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764762.008623:0:63201:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764762.008641:0:63201:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764762.008648:0:63201:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24a.sanity-pcc, flags=267 00000080:00200000:1.0:1775764762.008655:0:63201:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), inode mode 41ff mask 44 00000080:00200008:1.0:1775764762.008660:0:63201:0:(file.c:1057:ll_file_open()) START file d24a.sanity-pcc:[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), flags 304000 00000002:00010000:1.0:1775764762.008675:0:63201:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xb:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:1.0:1775764762.008725:0:63201:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17be79 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 63201 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764762.008732:0:63201:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17be79 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63201 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764762.008743:0:63201:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764762.008755:0:63201:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fc380 x1862022098626176/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764762.008764:0:63201:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17be79 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63201 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764762.008787:0:63201:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63201:1862022098626176:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764762.009023:0:63201:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238a33ac00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764762.038899:0:63201:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63201:1862022098626176:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764762.038922:0:63201:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17be79 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63201 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.038935:0:63201:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17be79 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 63201 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.038942:0:63201:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17be79 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 63201 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.038949:0:63201:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17be79 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 63201 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.038958:0:63201:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17be79 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 63201 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.038964:0:63201:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8741c00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764762.038969:0:63201:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17be79 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 63201 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.038976:0:63201:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17be79 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 63201 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.038983:0:63201:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17be79 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 63201 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.038995:0:63201:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764762.039001:0:63201:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17be79 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 63201 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.039007:0:63201:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17be79 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 63201 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764762.039025:0:63201:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23864fc380 x1862022098626176/t34359739189(34359739189) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764778 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764762.039038:0:63201:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fc380 x1862022098626176/t34359739189(34359739189) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764778 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764762.039050:0:63201:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764762.039069:0:63201:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xb:0x0], updating i_size 512 00000002:00100000:2.0:1775764762.039087:0:63201:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fc380 x1862022098626176/t34359739189(34359739189) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764778 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764762.039194:0:63201:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8) cmd=901066fc arg=7ffe171ba590 00000100:00100000:2.0:1775764762.039264:0:63201:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63201:1862022098626304:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764762.039353:0:63201:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0e2d0c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764762.044836:0:63201:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63201:1862022098626304:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764762.044920:0:63201:0:(file.c:447:ll_file_release()) START file d24a.sanity-pcc:[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), flags 304000 00000002:00100000:2.0:1775764762.044976:0:63201:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fc380 x1862022098626176/t34359739189(34359739189) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764778 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764762.044987:0:63201:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764762.044992:0:63201:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7480 x1862022098626560/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764762.045014:0:63201:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63201:1862022098626560:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764762.045083:0:63201:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0e2d0c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764762.055064:0:63201:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63201:1862022098626560:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764762.055103:0:63201:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fc380 x1862022098626176/t34359739189(34359739189) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764778 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764762.255538:0:63207:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.255551:0:63207:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/12 (0 0) 00000080:00200000:2.0:1775764762.255557:0:63207:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab4:0xb:0x0] suppgids -1 -1: rc 0 00000080:00200000:2.0:1775764762.255653:0:63207:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0xc:0x0](ffff8b23899c2a08): rc = 0 00000080:00200000:2.0:1775764762.255686:0:63207:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0xc:0x0]: layout version change: 2 -> 2 00000080:00010000:2.0:1775764762.255700:0:63207:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xc:0x0](ffff8b23899c2a08) for lock 0x73cfd86cfc17be80 00000080:00010000:2.0:1775764762.255711:0:63207:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xc:0x0](ffff8b23899c2a08) 00000080:00200008:2.0:1775764762.255724:0:63207:0:(file.c:1057:ll_file_open()) START file f24a.sanity-pcc:[0x200003ab4:0xc:0x0](ffff8b23899c2a08), flags 100000 00000002:00100000:2.0:1775764762.255745:0:63207:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864f3800 x1862022098626944/t34359739191(34359739191) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764778 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.500' uid:500 gid:500 projid:0 00000080:00010000:2.0:1775764762.255787:0:63207:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be80 from it ffff8b2382b00ae0 00010000:00010000:2.0:1775764762.255792:0:63207:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.255802:0:63207:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764762.255832:0:63207:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xc:0x0](ffff8b23899c2a08), inode mode 81a4 mask 44 00000080:00200008:2.0:1775764762.255878:0:63207:0:(file.c:5170:ll_file_seek()) START file f24a.sanity-pcc:[0x200003ab4:0xc:0x0], offset: 0, type: SEEK_CUR 00000080:00200008:2.0:1775764762.255883:0:63207:0:(file.c:5201:ll_file_seek()) COMPLETED file f24a.sanity-pcc:[0x200003ab4:0xc:0x0], offset: 0, type: SEEK_CUR, rc = 0 00000080:00200008:2.0:1775764762.255987:0:63207:0:(file.c:2454:do_file_read_iter()) START file f24a.sanity-pcc:[0x200003ab4:0xc:0x0], ppos: 0, count: 1024 00000080:00200008:2.0:1775764762.256048:0:63207:0:(file.c:447:ll_file_release()) START file f24a.sanity-pcc:[0x200003ab4:0xc:0x0](ffff8b23899c2a08), flags 100000 00000002:00100000:2.0:1775764762.256090:0:63207:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864f3800 x1862022098626944/t34359739191(34359739191) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764778 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.500' uid:500 gid:500 projid:0 00000020:00100000:2.0:1775764762.256100:0:63207:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764762.256104:0:63207:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7480 x1862022098627200/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764762.256125:0:63207:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:63207:1862022098627200:192.168.202.128@tcp:35:dd.500 00000100:00100000:2.0:1775764762.256569:0:63207:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0e2d0c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764762.261337:0:63207:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:63207:1862022098627200:192.168.202.128@tcp:35:dd.500 00000002:00100000:2.0:1775764762.261383:0:63207:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864f3800 x1862022098626944/t34359739191(34359739191) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764778 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.500' uid:500 gid:500 projid:0 00000100:00100000:2.0:1775764762.415652:0:63217:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63217:1862022098627456:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764762.415689:0:63217:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17be87 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63217 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.415702:0:63217:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17be87 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 63217 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.415710:0:63217:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17be87 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 63217 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.415717:0:63217:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17be87 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 63217 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.415729:0:63217:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17be87 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 63217 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.415735:0:63217:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23c2044600 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764762.415738:0:63217:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17be87 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 63217 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.415744:0:63217:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17be87 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 63217 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.415751:0:63217:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17be87 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 63217 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.415757:0:63217:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764762.415763:0:63217:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17be87 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 63217 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.415769:0:63217:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17be87 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 63217 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764762.415786:0:63217:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b2382dafb80 x1862022098627456/t34359739193(34359739193) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764778 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764762.415799:0:63217:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382dafb80 x1862022098627456/t34359739193(34359739193) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764778 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764762.415812:0:63217:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764762.415832:0:63217:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xb:0x0], updating i_size 512 00000002:00100000:2.0:1775764762.415863:0:63217:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382dafb80 x1862022098627456/t34359739193(34359739193) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764778 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764762.415957:0:63217:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8) cmd=901066fc arg=7ffed7c2fbd0 00000080:00200008:2.0:1775764762.415993:0:63217:0:(file.c:447:ll_file_release()) START file d24a.sanity-pcc:[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), flags 304000 00000002:00100000:2.0:1775764762.416045:0:63217:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b2382dafb80 x1862022098627456/t34359739193(34359739193) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764778 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764762.416055:0:63217:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764762.416060:0:63217:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7480 x1862022098627712/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764762.416082:0:63217:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63217:1862022098627712:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764762.416160:0:63217:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0e2d0c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764762.426689:0:63217:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63217:1862022098627712:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764762.426732:0:63217:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b2382dafb80 x1862022098627456/t34359739193(34359739193) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764778 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:1.0:1775764762.560530:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1775764762.638802:0:63223:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764762.638815:0:63223:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764762.638837:0:63223:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764762.638864:0:63223:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1775764762.638882:0:63223:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1775764762.638891:0:63223:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:2.0:1775764762.638901:0:63223:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764762.638905:0:63223:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:2.0:1775764762.638913:0:63223:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01f0bcb8 00010000:00010000:2.0:1775764762.638917:0:63223:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1775764762.638927:0:63223:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:2.0:1775764762.638955:0:63223:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764762.638961:0:63223:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764762.638966:0:63223:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:2.0:1775764762.638974:0:63223:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:2.0:1775764762.638980:0:63223:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:2.0:1775764762.638984:0:63223:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764762.638986:0:63223:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:2.0:1775764762.638989:0:63223:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01f0bb58 00010000:00010000:2.0:1775764762.638992:0:63223:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:2.0:1775764762.639000:0:63223:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:2.0:1775764762.639008:0:63223:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764762.639014:0:63223:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24a.sanity-pcc, flags=8 00000080:00200008:2.0:1775764762.639021:0:63223:0:(file.c:6226:ll_getattr_dentry()) START file d24a.sanity-pcc:[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764762.639025:0:63223:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8),name=d24a.sanity-pcc 00000002:00010000:2.0:1775764762.639036:0:63223:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xb:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764762.639043:0:63223:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764762.639050:0:63223:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764762.639056:0:63223:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:2.0:1775764762.639059:0:63223:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xb:0x0](ffff8b2390ec00c8) 00000080:00010000:2.0:1775764762.639061:0:63223:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xb:0x0](ffff8b2390ec00c8) for lock 0x73cfd86cfc17be64 00000080:00010000:2.0:1775764762.639064:0:63223:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be64 from it ffffa60b01f0bcb8 00010000:00010000:2.0:1775764762.639067:0:63223:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764762.639073:0:63223:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764762.639086:0:63223:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764762.639090:0:63223:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764762.639096:0:63223:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764762.639102:0:63223:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:2.0:1775764762.639108:0:63223:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:2.0:1775764762.639111:0:63223:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764762.639114:0:63223:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:2.0:1775764762.639116:0:63223:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01f0bb58 00010000:00010000:2.0:1775764762.639119:0:63223:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764762.639125:0:63223:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:2.0:1775764762.639131:0:63223:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764762.640005:0:63223:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24a.sanity-pcc, flags=24 00010000:00010000:2.0:1775764762.648901:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764762.648908:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17be8e lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0305213c expref: -99 pid: 63223 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.648916:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5fe00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764762.648921:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17be8e lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0305213c expref: -99 pid: 63223 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.648929:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17be8e lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e0305213c expref: -99 pid: 63223 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.648937:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17be8e lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0305213c expref: -99 pid: 63223 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.648955:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764762.648985:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17be8e lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305213c expref: -99 pid: 63223 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.648996:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764762.649018:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17be8e lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305213c expref: -99 pid: 63223 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.649025:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17be8e lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305213c expref: -99 pid: 63223 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764762.649039:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764762.649040:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764762.649042:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764762.649053:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098628480:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764762.654660:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098628480:192.168.202.128@tcp:103:ldlm_bl.0 00000080:00200000:1.0:1775764762.832313:0:63233:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764762.832345:0:63233:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764762.832368:0:63233:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764762.832385:0:63233:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1775764762.832392:0:63233:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764762.832399:0:63233:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764762.832403:0:63233:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:1.0:1775764762.832409:0:63233:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01f4fbf8 00010000:00010000:1.0:1775764762.832412:0:63233:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764762.832421:0:63233:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:1.0:1775764762.832437:0:63233:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764762.832444:0:63233:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24a.sanity-pcc, flags=267 00000080:00200000:1.0:1775764762.832651:0:63233:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), inode mode 41ff mask 44 00000080:00200008:1.0:1775764762.832660:0:63233:0:(file.c:1057:ll_file_open()) START file d24a.sanity-pcc:[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), flags 304000 00000002:00010000:1.0:1775764762.832676:0:63233:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xb:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:1.0:1775764762.832733:0:63233:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be95 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 63233 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764762.832741:0:63233:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be95 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63233 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764762.832753:0:63233:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764762.832767:0:63233:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd500 x1862022098628736/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764762.832782:0:63233:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be95 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63233 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764762.832807:0:63233:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63233:1862022098628736:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764762.833051:0:63233:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238a33ae40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764762.839386:0:63233:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63233:1862022098628736:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764762.839411:0:63233:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be95 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63233 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764762.839424:0:63233:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be95 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 63233 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764762.839431:0:63233:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be95 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 63233 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764762.839437:0:63233:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be95 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 63233 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764762.839446:0:63233:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be95 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 63233 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764762.839482:0:63233:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8740200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764762.839487:0:63233:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be95 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 63233 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764762.839494:0:63233:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be95 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 63233 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764762.839501:0:63233:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be95 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 63233 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764762.839510:0:63233:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764762.839517:0:63233:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be95 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 63233 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764762.839523:0:63233:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be95 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 63233 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764762.839543:0:63233:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23864fd500 x1862022098628736/t34359739197(34359739197) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764778 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764762.839556:0:63233:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fd500 x1862022098628736/t34359739197(34359739197) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764778 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764762.839569:0:63233:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764762.839590:0:63233:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xb:0x0], updating i_size 512 00000002:00100000:1.0:1775764762.839608:0:63233:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fd500 x1862022098628736/t34359739197(34359739197) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764778 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764762.839698:0:63233:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8) cmd=901066fc arg=7ffda5b46f40 00000080:00200008:1.0:1775764762.839732:0:63233:0:(file.c:447:ll_file_release()) START file d24a.sanity-pcc:[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), flags 304000 00000002:00100000:1.0:1775764762.839784:0:63233:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fd500 x1862022098628736/t34359739197(34359739197) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764778 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764762.839793:0:63233:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764762.839798:0:63233:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ff100 x1862022098628992/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764762.839818:0:63233:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63233:1862022098628992:192.168.202.128@tcp:35:lfs.0 00000100:00100000:1.0:1775764762.839907:0:63233:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238a33ae40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764762.842766:0:63233:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63233:1862022098628992:192.168.202.128@tcp:35:lfs.0 00000002:00100000:1.0:1775764762.842799:0:63233:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fd500 x1862022098628736/t34359739197(34359739197) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764778 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764762.948193:0:63236:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764762.948223:0:63236:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764762.948242:0:63236:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1775764762.948268:0:63236:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1775764762.948276:0:63236:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764762.948284:0:63236:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764762.948287:0:63236:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:2.0:1775764762.948294:0:63236:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01e5bb98 00010000:00010000:2.0:1775764762.948297:0:63236:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1775764762.948305:0:63236:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1775764762.948321:0:63236:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764762.948328:0:63236:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24a.sanity-pcc, flags=24 00000080:00200000:2.0:1775764762.948333:0:63236:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), inode mode 41ff mask 1 00000080:00200000:2.0:1775764762.948337:0:63236:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f24a.sanity-pcc, flags=8 00010000:00010000:1.0:1775764763.575659:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764763.740398:0:63270:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63270:1862022098629120:192.168.202.128@tcp:49:lfs.500 00000080:00200000:1.0:1775764763.740444:0:63270:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f24a.sanity-pcc, flags=8 00000080:00200008:1.0:1775764763.740484:0:63270:0:(file.c:6226:ll_getattr_dentry()) START file f24a.sanity-pcc:[0x200003ab4:0xc:0x0](ffff8b23899c2a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764763.740492:0:63270:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0xc:0x0](ffff8b23899c2a08),name=f24a.sanity-pcc 00000002:00010000:1.0:1775764763.740516:0:63270:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xc:0x0]) in obj [0x200003ab4:0xb:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764763.740540:0:63270:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764763.740554:0:63270:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764763.740562:0:63270:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200003ab4:0xb:0x0] suppgids -1 500: rc 1 00000080:00010000:1.0:1775764763.740570:0:63270:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xc:0x0](ffff8b23899c2a08) 00000080:00010000:1.0:1775764763.740573:0:63270:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xc:0x0](ffff8b23899c2a08) for lock 0x73cfd86cfc17be80 00000080:00010000:1.0:1775764763.740580:0:63270:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be80 from it ffffa60b01fefcb8 00010000:00010000:1.0:1775764763.740584:0:63270:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764763.740594:0:63270:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764763.740647:0:63270:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0xc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1775764763.740660:0:63270:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0xc:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764763.740664:0:63270:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1775764763.740668:0:63270:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0xc:0x0] 00000008:00010000:1.0:1775764763.740697:0:63270:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764763.740700:0:63270:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5eb0, osc lock ffff8b2382cf6130, flags 0x400000400001002 00010000:00010000:1.0:1775764763.740719:0:63270:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1775764763.740732:0:63270:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000020:00200000:1.0:1775764763.740744:0:63270:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764763.740747:0:63270:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1775764763.740756:0:63270:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000020:00200000:1.0:1775764763.740767:0:63270:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764763.740778:0:63270:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0xc:0x0] updating i_size 1024 i_blocks 1 00000020:00010000:1.0:1775764763.740781:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_release():245 00000020:00010000:1.0:1775764763.740784:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764763.740787:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5eb0 (ffff8b2387cba008/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764763.740790:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764763.740793:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5eb0 (ffff8b2387cba368/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764763.740834:0:63270:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xc:0x0] ignore/verify layout 0/1, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764763.740886:0:63270:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764763.740895:0:63270:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764763.740903:0:63270:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1775764763.740912:0:63270:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:1.0:1775764763.740918:0:63270:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:1.0:1775764763.740923:0:63270:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764763.740925:0:63270:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:1.0:1775764763.740929:0:63270:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01fefbf8 00010000:00010000:1.0:1775764763.740932:0:63270:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1775764763.740938:0:63270:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:1.0:1775764763.740946:0:63270:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764763.740951:0:63270:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24a.sanity-pcc, flags=25 00000080:00200000:1.0:1775764763.740955:0:63270:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), inode mode 41ff mask 1 00000080:00200000:1.0:1775764763.740958:0:63270:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f24a.sanity-pcc, flags=265 00000080:00200000:1.0:1775764763.740963:0:63270:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xc:0x0](ffff8b23899c2a08), inode mode 81a4 mask 44 00000080:00200008:1.0:1775764763.740970:0:63270:0:(file.c:1057:ll_file_open()) START file f24a.sanity-pcc:[0x200003ab4:0xc:0x0](ffff8b23899c2a08), flags 100000 00000002:00010000:1.0:1775764763.740984:0:63270:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xc:0x0]) in obj [0x200003ab4:0xb:0x0], intent: open flags 04240100001 00010000:00010000:1.0:1775764763.741037:0:63270:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be9c lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 63270 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764763.741045:0:63270:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be9c lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 63270 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764763.741055:0:63270:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764763.741064:0:63270:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ff100 x1862022098629376/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764763.741074:0:63270:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be9c lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 63270 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764763.741095:0:63270:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63270:1862022098629376:192.168.202.128@tcp:101:lfs.500 00000100:00100000:1.0:1775764763.741498:0:63270:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238a33ae40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764763.749856:0:63270:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63270:1862022098629376:192.168.202.128@tcp:101:lfs.500 00010000:00010000:1.0:1775764763.749899:0:63270:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be9c lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305215f expref: -99 pid: 63270 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764763.749912:0:63270:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be9c lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305215f expref: -99 pid: 63270 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764763.749926:0:63270:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be9c lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305215f expref: -99 pid: 63270 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764763.749938:0:63270:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be9c lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305215f expref: -99 pid: 63270 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764763.749955:0:63270:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23864ff100 x1862022098629376/t34359739199(34359739199) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764779 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.500' uid:500 gid:500 projid:0 00000002:00100000:1.0:1775764763.750099:0:63270:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864ff100 x1862022098629376/t34359739199(34359739199) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764779 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.500' uid:500 gid:500 projid:0 00000002:00010000:1.0:1775764763.750113:0:63270:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be9c lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305215f expref: -99 pid: 63270 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764763.750141:0:63270:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be9c lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305215f expref: -99 pid: 63270 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764763.750153:0:63270:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/12 (0 0) 00000080:00200000:1.0:1775764763.750160:0:63270:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab4:0xb:0x0] suppgids -1 500: rc 0 00000080:00200000:1.0:1775764763.750384:0:63270:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0xc:0x0]: layout version change: 2 -> 2 00000080:00010000:1.0:1775764763.750395:0:63270:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xc:0x0](ffff8b23899c2a08) for lock 0x73cfd86cfc17be9c 00000080:00010000:1.0:1775764763.750402:0:63270:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be9c from it ffffa60b01fefbf0 00010000:00010000:1.0:1775764763.750405:0:63270:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be9c lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305215f expref: -99 pid: 63270 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764763.750415:0:63270:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be9c lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305215f expref: -99 pid: 63270 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764763.750440:0:63270:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864ff100 x1862022098629376/t34359739199(34359739199) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764779 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.500' uid:500 gid:500 projid:0 00000080:00200080:1.0:1775764763.750666:0:63270:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0xc:0x0](0000000000000000) cmd=400866fc arg=7ffc6770c2b8 00000080:00200008:1.0:1775764763.751613:0:63270:0:(file.c:2454:do_file_read_iter()) START file f24a.sanity-pcc:[0x200003ab4:0xc:0x0], ppos: 0, count: 1048576 00000080:00200000:1.0:1775764763.751651:0:63270:0:(file.c:2058:ll_file_io_generic()) f24a.sanity-pcc: read ppos: 1024, bytes: 1047552 00000080:00200000:1.0:1775764763.751657:0:63270:0:(file.c:1815:ll_io_set_mirror()) f24a.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764763.751665:0:63270:0:(cl_io.c:214:cl_io_rw_init()) header@0000000042a321a3[0x110, 2, [0x200003ab4:0xc:0x0] hash] 00000020:00200000:1.0:1775764763.751667:0:63270:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [1024, 1048576) 0 0 00000080:00200000:1.0:1775764763.751672:0:63270:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764763.751686:0:63270:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0xc:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764763.751690:0:63270:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:1.0:1775764763.751698:0:63270:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 1024 chunk: [1024, 1048576) 1048576, 1047552 00020000:00200000:1.0:1775764763.751702:0:63270:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764763.751732:0:63270:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:1.0:1775764763.751745:0:63270:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764763.751749:0:63270:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5eb0, osc lock ffff8b2382cf6130, flags 0x400000000000000 00010000:00010000:1.0:1775764763.751769:0:63270:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764763.751783:0:63270:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764763.751794:0:63270:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764763.753156:0:63270:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764763.753165:0:63270:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000080:00200000:1.0:1775764763.753177:0:63270:0:(vvp_io.c:826:vvp_io_read_start()) f24a.sanity-pcc: read [1024, 1048576) 00000001:00010000:1.0:1775764763.753196:0:63270:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0xc:0x0] 00000008:00010000:1.0:1775764763.753206:0:63270:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764763.753298:0:63270:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c56b0, osc lock ffff8b2382cf6e40, flags 0x400000400001002 00010000:00010000:1.0:1775764763.753313:0:63270:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764763.753325:0:63270:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000020:00200000:1.0:1775764763.753341:0:63270:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764763.753345:0:63270:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764763.753352:0:63270:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 6/1,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000020:00200000:1.0:1775764763.753359:0:63270:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764763.753367:0:63270:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0xc:0x0] updating i_size 1024 i_blocks 1 00000020:00010000:1.0:1775764763.753371:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_release():245 00000020:00010000:1.0:1775764763.753374:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764763.753377:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c56b0 (ffff8b2387cba008/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764763.753382:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764763.753384:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c56b0 (ffff8b2387cba368/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764763.753396:0:63270:0:(vvp_io.c:885:vvp_io_read_start()) count:1047552 iocb pos:1024 00000020:00010000:1.0:1775764763.753408:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb7b78 (ffff8b2387cba368/0) at cl_lock_release():245 00000020:00010000:1.0:1775764763.753411:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb7b78 (ffff8b2387cba368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764763.753414:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5eb0 (ffff8b2387cba008/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764763.753417:0:63270:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1775764763.753425:0:63270:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00010000:1.0:1775764763.753439:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb7b78 (ffff8b2387cba368/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764763.753441:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5eb0 (ffff8b2387cba368/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764763.753448:0:63270:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:1.0:1775764763.753499:0:63270:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xc:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764763.753503:0:63270:0:(file.c:2206:ll_file_io_generic()) f24a.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:1.0:1775764763.753506:0:63270:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:1.0:1775764763.753568:0:63270:0:(file.c:2454:do_file_read_iter()) START file f24a.sanity-pcc:[0x200003ab4:0xc:0x0], ppos: 1024, count: 1048576 00000080:00200000:1.0:1775764763.753590:0:63270:0:(file.c:2058:ll_file_io_generic()) f24a.sanity-pcc: read ppos: 1024, bytes: 1048576 00000080:00200000:1.0:1775764763.753594:0:63270:0:(file.c:1815:ll_io_set_mirror()) f24a.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764763.753599:0:63270:0:(cl_io.c:214:cl_io_rw_init()) header@0000000042a321a3[0x110, 2, [0x200003ab4:0xc:0x0] hash] 00000020:00200000:1.0:1775764763.753601:0:63270:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [1024, 1049600) 0 0 00000080:00200000:1.0:1775764763.753604:0:63270:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0xc:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764763.753614:0:63270:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0xc:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764763.753616:0:63270:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:1.0:1775764763.753622:0:63270:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 1024 chunk: [1024, 1049600) 1049600, 1048576 00020000:00200000:1.0:1775764763.753625:0:63270:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764763.753647:0:63270:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 256] 00000008:00010000:1.0:1775764763.753654:0:63270:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764763.753657:0:63270:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5eb0, osc lock ffff8b2382cf6130, flags 0x400000000000000 00010000:00010000:1.0:1775764763.753665:0:63270:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764763.753674:0:63270:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1052671) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764763.753682:0:63270:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764763.753689:0:63270:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764763.753700:0:63270:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000080:00200000:1.0:1775764763.753706:0:63270:0:(vvp_io.c:826:vvp_io_read_start()) f24a.sanity-pcc: read [1024, 1049600) 00000001:00010000:1.0:1775764763.753714:0:63270:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0xc:0x0] 00000008:00010000:1.0:1775764763.753718:0:63270:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764763.753719:0:63270:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c56b0, osc lock ffff8b2382cf6e40, flags 0x400000400001002 00010000:00010000:1.0:1775764763.753725:0:63270:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764763.753733:0:63270:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00200000:1.0:1775764763.753739:0:63270:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764763.753742:0:63270:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764763.753748:0:63270:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 6/1,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00200000:1.0:1775764763.753755:0:63270:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764763.753759:0:63270:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0xc:0x0] updating i_size 1024 i_blocks 1 00000020:00010000:1.0:1775764763.753761:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_release():245 00000020:00010000:1.0:1775764763.753763:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764763.753765:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c56b0 (ffff8b2387cba008/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764763.753767:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764763.753770:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c56b0 (ffff8b2387cba368/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764763.753775:0:63270:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:1024 00000020:00010000:1.0:1775764763.753779:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb7b78 (ffff8b2387cba368/0) at cl_lock_release():245 00000020:00010000:1.0:1775764763.753782:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb7b78 (ffff8b2387cba368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764763.753784:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5eb0 (ffff8b2387cba008/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764763.753788:0:63270:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1775764763.755536:0:63270:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000fc8b66f6/0x73cfd86cfc17be5d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030520a2 expref: -99 pid: 63185 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000020:00010000:1.0:1775764763.755570:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb7b78 (ffff8b2387cba368/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764763.755573:0:63270:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5eb0 (ffff8b2387cba368/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764763.755588:0:63270:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:1.0:1775764763.755627:0:63270:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xc:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764763.755631:0:63270:0:(file.c:2206:ll_file_io_generic()) f24a.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:1.0:1775764763.755634:0:63270:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:1.0:1775764763.757512:0:63270:0:(file.c:447:ll_file_release()) START file f24a.sanity-pcc:[0x200003ab4:0xc:0x0](ffff8b23899c2a08), flags 100000 00010000:00010000:1.0:1775764763.757547:0:63270:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be9c lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305215f expref: -99 pid: 63270 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000100:00100000:1.0:1775764763.938101:0:63280:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63280:1862022098629632:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764763.938139:0:63280:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bea3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764763.938153:0:63280:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bea3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 63280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764763.938166:0:63280:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bea3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 63280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764763.938173:0:63280:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bea3 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 63280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764763.938182:0:63280:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bea3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 63280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764763.938189:0:63280:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23c2045600 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764763.938193:0:63280:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bea3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 63280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764763.938201:0:63280:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bea3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 63280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764763.938208:0:63280:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bea3 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 63280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764763.938215:0:63280:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764763.938221:0:63280:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bea3 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 63280 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764763.938426:0:63280:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bea3 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 63280 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764763.938448:0:63280:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b2382daf480 x1862022098629632/t34359739200(34359739200) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764779 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764763.938591:0:63280:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382daf480 x1862022098629632/t34359739200(34359739200) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764779 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764763.938606:0:63280:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764763.938630:0:63280:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xb:0x0], updating i_size 7680 00000002:00100000:1.0:1775764763.938650:0:63280:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382daf480 x1862022098629632/t34359739200(34359739200) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764779 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764763.938748:0:63280:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8) cmd=901066fc arg=7ffcf009ecb0 00000080:00200008:1.0:1775764763.938785:0:63280:0:(file.c:447:ll_file_release()) START file d24a.sanity-pcc:[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), flags 304000 00000002:00100000:1.0:1775764763.940178:0:63280:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b2382daf480 x1862022098629632/t34359739200(34359739200) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764779 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764763.940192:0:63280:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764763.940198:0:63280:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd500 x1862022098629888/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764764.002086:0:63280:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 34359739199 gen 1 00000100:00100000:2.0:1775764764.002104:0:63280:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23864fce00 x1862022098622336/t34359739181(34359739181) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/4688 e 0 to 0 dl 1775764776 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764764.002143:0:63280:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23864ff100 x1862022098629376/t34359739199(34359739199) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764779 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.500' uid:500 gid:500 projid:0 00000100:00100000:2.0:1775764764.002153:0:63280:0:(client.c:3126:ptlrpc_free_committed()) @@@ stopping search req@ffff8b2382daf480 x1862022098629632/t34359739200(34359739200) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764779 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764764.002163:0:63280:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63280:1862022098629888:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764764.002191:0:63280:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b2382daf480 x1862022098629632/t34359739200(34359739200) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764779 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1775764764.146826:0:63286:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764764.146840:0:63286:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764764.146875:0:63286:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764764.146898:0:63286:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:1.0:1775764764.146915:0:63286:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:1.0:1775764764.146923:0:63286:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:1.0:1775764764.146931:0:63286:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764764.146935:0:63286:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:1.0:1775764764.146942:0:63286:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b02037cb8 00010000:00010000:1.0:1775764764.146946:0:63286:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:1.0:1775764764.146956:0:63286:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:1.0:1775764764.146981:0:63286:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764764.146988:0:63286:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764764.146994:0:63286:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:1.0:1775764764.147003:0:63286:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:1.0:1775764764.147009:0:63286:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:1.0:1775764764.147014:0:63286:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764764.147016:0:63286:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:1.0:1775764764.147020:0:63286:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b02037b58 00010000:00010000:1.0:1775764764.147023:0:63286:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:1.0:1775764764.149650:0:63286:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:1.0:1775764764.149669:0:63286:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764764.149683:0:63286:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24a.sanity-pcc, flags=8 00000080:00200008:1.0:1775764764.149697:0:63286:0:(file.c:6226:ll_getattr_dentry()) START file d24a.sanity-pcc:[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764764.149702:0:63286:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8),name=d24a.sanity-pcc 00000002:00010000:1.0:1775764764.149728:0:63286:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xb:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764764.149746:0:63286:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764764.149757:0:63286:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764764.149765:0:63286:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:1.0:1775764764.149773:0:63286:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xb:0x0](ffff8b2390ec00c8) 00000080:00010000:1.0:1775764764.149776:0:63286:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xb:0x0](ffff8b2390ec00c8) for lock 0x73cfd86cfc17be64 00000080:00010000:1.0:1775764764.149782:0:63286:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be64 from it ffffa60b02037cb8 00010000:00010000:1.0:1775764764.149786:0:63286:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764764.149793:0:63286:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764764.149824:0:63286:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764764.149831:0:63286:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764764.149837:0:63286:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:1.0:1775764764.149845:0:63286:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:1.0:1775764764.149851:0:63286:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:1.0:1775764764.149855:0:63286:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764764.149858:0:63286:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:1.0:1775764764.149874:0:63286:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b02037b58 00010000:00010000:1.0:1775764764.149878:0:63286:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:1.0:1775764764.149885:0:63286:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:1.0:1775764764.149893:0:63286:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764764.149898:0:63286:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24a.sanity-pcc, flags=24 00000080:00200000:1.0:1775764764.149901:0:63286:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), inode mode 41ff mask 1 00000100:00100000:1.0:1775764764.149975:0:63286:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63286:1862022098630016:192.168.202.128@tcp:49:lfs.500 00000100:00100000:1.0:1775764764.150087:0:63286:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238a33a000 going to sleep for 11 seconds 00000100:00100000:1.0:1775764764.160400:0:63286:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63286:1862022098630016:192.168.202.128@tcp:49:lfs.500 00000080:00200000:1.0:1775764764.160442:0:63286:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f24a.sanity-pcc, flags=8 00000080:00200008:1.0:1775764764.160494:0:63286:0:(file.c:6226:ll_getattr_dentry()) START file f24a.sanity-pcc:[0x200003ab4:0xc:0x0](ffff8b23899c2a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764764.160501:0:63286:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0xc:0x0](ffff8b23899c2a08),name=f24a.sanity-pcc 00000002:00010000:1.0:1775764764.160526:0:63286:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xc:0x0]) in obj [0x200003ab4:0xb:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764764.160545:0:63286:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764764.160560:0:63286:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764764.160568:0:63286:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200003ab4:0xb:0x0] suppgids -1 500: rc 1 00000080:00010000:1.0:1775764764.160575:0:63286:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xc:0x0](ffff8b23899c2a08) 00000080:00010000:1.0:1775764764.160579:0:63286:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xc:0x0](ffff8b23899c2a08) for lock 0x73cfd86cfc17be80 00000080:00010000:1.0:1775764764.160586:0:63286:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be80 from it ffffa60b02037cb8 00010000:00010000:1.0:1775764764.160590:0:63286:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764764.160602:0:63286:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764764.160661:0:63286:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764764.160670:0:63286:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764764.160677:0:63286:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764764.160686:0:63286:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:1.0:1775764764.160692:0:63286:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:1.0:1775764764.160697:0:63286:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764764.160699:0:63286:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:1.0:1775764764.160702:0:63286:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b02037bf8 00010000:00010000:1.0:1775764764.160705:0:63286:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764764.160712:0:63286:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:1.0:1775764764.160734:0:63286:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764764.160740:0:63286:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24a.sanity-pcc, flags=25 00000080:00200000:1.0:1775764764.160745:0:63286:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), inode mode 41ff mask 1 00000080:00200000:1.0:1775764764.160748:0:63286:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f24a.sanity-pcc, flags=265 00000080:00200000:1.0:1775764764.160752:0:63286:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xc:0x0](ffff8b23899c2a08), inode mode 81a4 mask 46 00000080:00200008:1.0:1775764764.160760:0:63286:0:(file.c:1057:ll_file_open()) START file f24a.sanity-pcc:[0x200003ab4:0xc:0x0](ffff8b23899c2a08), flags 154402 00000002:00010000:1.0:1775764764.160773:0:63286:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xc:0x0]) in obj [0x200003ab4:0xb:0x0], intent: open flags 0104240114403 00010000:00010000:2.0:1775764764.162760:0:63286:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17beaa lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 63286 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764764.162774:0:63286:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17beaa lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 63286 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764764.162786:0:63286:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764764.162797:0:63286:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd500 x1862022098630272/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764764.162808:0:63286:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17beaa lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 63286 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764764.162832:0:63286:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63286:1862022098630272:192.168.202.128@tcp:101:lfs.500 00000100:00100000:2.0:1775764764.163039:0:63286:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0e2d0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764764.182646:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119057664 00000100:00100000:1.0:1775764764.182656:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119057664 00000100:00100000:1.0:1775764764.182730:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 254 00000100:00100000:1.0:1775764764.182740:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dac700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:44493:x1862022119057664:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764764.182760:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764764.182855:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764764.182862:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dac700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:44493:x1862022119057664:12345-192.168.202.128@tcp:104: Request processed in 124us (282us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764764.182869:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 254 00010000:00010000:2.0:1775764764.184819:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764764.184836:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764764.184849:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5fc00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764764.184856:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764764.184865:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764764.184875:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764764.184907:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be9c lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305215f expref: -99 pid: 63270 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764764.184916:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764764.184976:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764764.184988:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764764.185041:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764764.185049:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17be80 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0xc:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052119 expref: -99 pid: 63207 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764764.185068:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764764.185070:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764764.185072:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764764.185082:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764764.185103:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098630400:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764764.201899:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098630400:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764764.202057:0:63286:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63286:1862022098630272:192.168.202.128@tcp:101:lfs.500 00010000:00010000:1.0:1775764764.202079:0:63286:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CW ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17beaa lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052182 expref: -99 pid: 63286 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764764.202095:0:63286:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17beaa lrc: 4/1,0 mode: --/CW res: [0x200003ab4:0xc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052182 expref: -99 pid: 63286 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764764.202102:0:63286:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17beaa lrc: 4/1,0 mode: --/CW res: [0x200003ab4:0xc:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052182 expref: -99 pid: 63286 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764764.202113:0:63286:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17beaa lrc: 4/1,0 mode: CW/CW res: [0x200003ab4:0xc:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052182 expref: -99 pid: 63286 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764764.202137:0:63286:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17beaa lrc: 4/1,0 mode: CW/CW res: [0x200003ab4:0xc:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052182 expref: -99 pid: 63286 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764764.202156:0:63286:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17beaa lrc: 5/1,1 mode: CW/CW res: [0x200003ab4:0xc:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052182 expref: -99 pid: 63286 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764764.202165:0:63286:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17beaa lrc: 5/1,1 mode: CW/CW res: [0x200003ab4:0xc:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052182 expref: -99 pid: 63286 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764764.202172:0:63286:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17beaa lrc: 4/0,1 mode: CW/CW res: [0x200003ab4:0xc:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052182 expref: -99 pid: 63286 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764764.202182:0:63286:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23864fd500 x1862022098630272/t34359739202(34359739202) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764780 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.500' uid:500 gid:500 projid:0 00000002:00100000:1.0:1775764764.202195:0:63286:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fd500 x1862022098630272/t34359739202(34359739202) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764780 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.500' uid:500 gid:500 projid:0 00000002:00010000:1.0:1775764764.202210:0:63286:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17beaa lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0xc:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052182 expref: -99 pid: 63286 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764764.202221:0:63286:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 8 res 8589949620/12 (0 0) 00000080:00200000:1.0:1775764764.202246:0:63286:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab4:0xb:0x0] suppgids -1 500: rc 0 00000080:00010000:1.0:1775764764.202345:0:63286:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xc:0x0](ffff8b23899c2a08) for lock 0x73cfd86cfc17beaa 00000080:00010000:1.0:1775764764.202354:0:63286:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17beaa from it ffffa60b02037bf0 00010000:00010000:1.0:1775764764.202358:0:63286:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17beaa lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0xc:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052182 expref: -99 pid: 63286 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764764.202368:0:63286:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17beaa lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0xc:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052182 expref: -99 pid: 63286 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764764.202399:0:63286:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fd500 x1862022098630272/t34359739202(34359739202) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764780 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.500' uid:500 gid:500 projid:0 00000080:00200080:1.0:1775764764.202763:0:63286:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0xc:0x0](0000000000000000) cmd=c00466fc arg=7ffd73895d3c 00000080:00200008:1.0:1775764764.202842:0:63286:0:(file.c:447:ll_file_release()) START file f24a.sanity-pcc:[0x200003ab4:0xc:0x0](ffff8b23899c2a08), flags 154002 00010000:00010000:1.0:1775764764.202861:0:63286:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17beaa lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0xc:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052182 expref: -99 pid: 63286 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764764.362433:0:63297:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764764.362623:0:63297:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764764.362643:0:63297:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764764.362661:0:63297:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:1.0:1775764764.362669:0:63297:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764764.362679:0:63297:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764764.362682:0:63297:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:1.0:1775764764.362689:0:63297:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b0215fbf8 00010000:00010000:1.0:1775764764.362694:0:63297:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:1.0:1775764764.362702:0:63297:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:1.0:1775764764.362716:0:63297:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764764.362723:0:63297:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24a.sanity-pcc, flags=267 00000080:00200000:1.0:1775764764.362730:0:63297:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), inode mode 41ff mask 44 00000080:00200008:1.0:1775764764.362736:0:63297:0:(file.c:1057:ll_file_open()) START file d24a.sanity-pcc:[0x200003ab4:0xb:0x0](ffff8b2390ec00c8), flags 304000 00000002:00010000:1.0:1775764764.362752:0:63297:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xb:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240304001 00010000:00010000:1.0:1775764764.369756:0:63297:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740e00/0x73cfd86cfc17beb1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 63297 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764764.369775:0:63297:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740e00/0x73cfd86cfc17beb1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63297 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764764.369799:0:63297:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764764.369812:0:63297:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ffb80 x1862022098630656/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764764.369824:0:63297:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740e00/0x73cfd86cfc17beb1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63297 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764764.370105:0:63297:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63297:1862022098630656:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764764.370361:0:63297:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0e2d240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764764.382812:0:63297:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63297:1862022098630656:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764764.382845:0:63297:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740e00/0x73cfd86cfc17beb1 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052197 expref: -99 pid: 63297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764764.382856:0:63297:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740e00/0x73cfd86cfc17beb1 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052197 expref: -99 pid: 63297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764764.382868:0:63297:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740e00/0x73cfd86cfc17beb1 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052197 expref: -99 pid: 63297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764764.382883:0:63297:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740e00/0x73cfd86cfc17beb1 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052197 expref: -99 pid: 63297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764764.382898:0:63297:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23864ffb80 x1862022098630656/t34359739203(34359739203) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764780 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764764.382912:0:63297:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864ffb80 x1862022098630656/t34359739203(34359739203) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764780 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764764.382922:0:63297:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740e00/0x73cfd86cfc17beb1 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052197 expref: -99 pid: 63297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764764.382932:0:63297:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/11 (0 0) 00000080:00200000:2.0:1775764764.382938:0:63297:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764764.382960:0:63297:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xb:0x0], updating i_size 7680 00000080:00010000:2.0:1775764764.382965:0:63297:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xb:0x0](ffff8b2390ec00c8) for lock 0x73cfd86cfc17beb1 00000080:00010000:2.0:1775764764.382972:0:63297:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17beb1 from it ffffa60b0215fbd0 00010000:00010000:2.0:1775764764.382976:0:63297:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740e00/0x73cfd86cfc17beb1 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052197 expref: -99 pid: 63297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764764.382986:0:63297:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740e00/0x73cfd86cfc17beb1 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052197 expref: -99 pid: 63297 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:2.0:1775764764.383012:0:63297:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864ffb80 x1862022098630656/t34359739203(34359739203) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764780 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764764.383103:0:63297:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200003ab4:0xb:0x0](ffff8b2390ec00c8) cmd=901066fc arg=7ffe24244ef0 00000100:00100000:2.0:1775764764.383164:0:63297:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63297:1862022098630784:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764764.383257:0:63297:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0e2d240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764764.398777:0:63297:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:63297:1862022098630784:192.168.202.128@tcp:34:lfs.0 00010000:00010000:1.0:1775764764.602512:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764765.623725:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764765.803419:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2376:1862022098630912:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764765.803450:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098631168:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764765.803492:0:2376:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b2389398000: committing for last_committed 34359738451 gen 1 00000100:00100000:1.0:1775764765.803518:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098631424:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764765.803531:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2376:1862022098631040:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764765.803542:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098631296:192.168.202.128@tcp:400:kworker.0 00010000:00010000:2.0:1775764767.683635:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764768.713599:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764771.779716:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764772.791708:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764773.826224:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1775764773.984735:0:63726:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764773.984748:0:63726:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764773.984768:0:63726:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764773.984785:0:63726:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:1.0:1775764773.984801:0:63726:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:1.0:1775764773.984809:0:63726:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764773.984816:0:63726:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764773.984820:0:63726:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:1.0:1775764773.984827:0:63726:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01e33cb8 00010000:00010000:1.0:1775764773.984830:0:63726:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:1.0:1775764773.984839:0:63726:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200008:1.0:1775764773.992136:0:63726:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764773.992156:0:63726:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:1.0:1775764773.992179:0:63726:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764773.992239:0:63726:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 63726 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764773.992249:0:63726:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63726 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764773.992260:0:63726:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63726 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764773.992288:0:63726:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job lctl:82781e18-e0e3-4f9f-9a2b-041bf0344414:63726:1862022098632192:192.168.202.128@tcp:101:lctl.0 00000100:00100000:1.0:1775764773.992444:0:63726:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2384989780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764773.999736:0:63726:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job lctl:82781e18-e0e3-4f9f-9a2b-041bf0344414:63726:1862022098632192:192.168.202.128@tcp:101:lctl.0 00010000:00010000:2.0:1775764773.999763:0:63726:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.999789:0:63726:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.999799:0:63726:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.999810:0:63726:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.999978:0:63726:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.999988:0:63726:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764773.999994:0:63726:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764774.000012:0:63726:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864fc380 x1862022098632192/t0(0) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764789 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lctl.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764774.000026:0:63726:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.000040:0:63726:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2387c07800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764774.000046:0:63726:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764774.000083:0:63726:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764774.000090:0:63726:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17beb8 00000080:00010000:2.0:1775764774.000096:0:63726:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:2.0:1775764774.000099:0:63726:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17beb8 from it ffffa60b01e33cb8 00010000:00010000:2.0:1775764774.000102:0:63726:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764774.000111:0:63726:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764774.000361:0:63726:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764774.000380:0:63726:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764774.000392:0:63726:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:2.0:1775764774.000402:0:63726:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:2.0:1775764774.000408:0:63726:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764774.000414:0:63726:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764774.000416:0:63726:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:2.0:1775764774.000420:0:63726:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01e33bb0 00010000:00010000:2.0:1775764774.000422:0:63726:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:2.0:1775764774.000429:0:63726:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:2.0:1775764774.000440:0:63726:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764774.000448:0:63726:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200080:2.0:1775764774.000510:0:63726:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=8080667f arg=7ffcf385eb30 00000080:00200008:2.0:1775764774.000529:0:63726:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000100:00100000:1.0:1775764774.185170:0:63168:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3800 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:63168:1862022098632576:192.168.202.128@tcp:35:lhsmtool_posix.0 00000002:00100000:1.0:1775764774.185205:0:63168:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fce00 x1862022098622336/t34359739181(34359739181) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/4688 e 0 to 0 dl 1775764776 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1775764774.185280:0:63168:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 100000 00000080:00200008:1.0:1775764774.194712:0:63168:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 100000 00010000:00010000:2.0:1775764774.839719:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764775.864068:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764776.006501:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098632832:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764776.006525:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098633088:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764776.006535:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098632704:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764776.006545:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098632960:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764776.006556:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098633216:192.168.202.128@tcp:400:kworker.0 00010000:00010000:2.0:1775764776.888963:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764777.925429:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764778.939281:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1775764779.908139:0:63903:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-pcc test 24b: General permission test for RO-PCC ========================================================== 15:59:38 (1775764778) 00010000:00010000:2.0:1775764780.983637:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764781.111693:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098633472:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764781.111796:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098633728:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764781.111832:0:2377:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:1.0:1775764781.111838:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098633344:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764781.111867:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098633600:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764781.111889:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098633856:192.168.202.128@tcp:400:kworker.0 00010000:00010000:2.0:1775764782.370523:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764782.370530:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bebf lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e030521dd expref: -99 pid: 64004 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.370537:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5e600 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764782.370541:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bebf lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e030521dd expref: -99 pid: 64004 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.370549:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bebf lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e030521dd expref: -99 pid: 64004 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.370556:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bebf lrc: 3/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e030521dd expref: -99 pid: 64004 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.370572:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e000/0x73cfd86cfc17baf9 lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305144d expref: -99 pid: 61408 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764782.370604:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bebf lrc: 2/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030521dd expref: -99 pid: 64004 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.370611:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764782.370641:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bebf lrc: 1/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030521dd expref: -99 pid: 64004 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.370648:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17bebf lrc: 0/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030521dd expref: -99 pid: 64004 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.370661:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764782.370663:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764782.370664:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764782.370678:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098634240:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764782.380792:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098634240:192.168.202.128@tcp:103:ldlm_bl.0 00000080:00200000:2.0:1775764782.426680:0:64012:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764782.426718:0:64012:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764782.426739:0:64012:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00010000:00010000:2.0:1775764782.426752:0:64012:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00000080:00200000:2.0:1775764782.426762:0:64012:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764782.426769:0:64012:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764782.426771:0:64012:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:2.0:1775764782.426776:0:64012:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b00dd3bb0 00010000:00010000:2.0:1775764782.426779:0:64012:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00010000:00010000:2.0:1775764782.426785:0:64012:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00000080:00200000:2.0:1775764782.426796:0:64012:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764782.426801:0:64012:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200080:2.0:1775764782.426821:0:64012:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=8080667f arg=7ffcc8162440 00000080:00200008:2.0:1775764782.426833:0:64012:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200000:2.0:1775764782.447389:0:64013:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764782.447414:0:64013:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764782.447433:0:64013:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00010000:00010000:2.0:1775764782.447449:0:64013:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00000080:00200000:2.0:1775764782.447695:0:64013:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764782.447703:0:64013:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764782.447707:0:64013:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:2.0:1775764782.447713:0:64013:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01dabe18 00010000:00010000:2.0:1775764782.447718:0:64013:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00010000:00010000:2.0:1775764782.447727:0:64013:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00000080:00200000:2.0:1775764782.447741:0:64013:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 101 00000080:00200000:2.0:1775764782.447752:0:64013:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764782.447758:0:64013:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764782.447764:0:64013:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:2.0:1775764782.447771:0:64013:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00000080:00200000:2.0:1775764782.447776:0:64013:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764782.447780:0:64013:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764782.447782:0:64013:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:2.0:1775764782.447785:0:64013:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01dabbe0 00010000:00010000:2.0:1775764782.447788:0:64013:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:2.0:1775764782.447795:0:64013:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:2.0:1775764782.447802:0:64013:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764782.447814:0:64013:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=d24b.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), flags=1536 00000080:00200000:2.0:1775764782.447817:0:64013:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764782.447823:0:64013:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764782.447828:0:64013:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:2.0:1775764782.447835:0:64013:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:2.0:1775764782.447852:0:64013:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764782.447856:0:64013:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764782.447858:0:64013:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:2.0:1775764782.447861:0:64013:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01dabce0 00010000:00010000:2.0:1775764782.447864:0:64013:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:2.0:1775764782.447870:0:64013:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:2.0:1775764782.447877:0:64013:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:2.0:1775764782.447882:0:64013:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764782.447888:0:64013:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764782.447893:0:64013:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:2.0:1775764782.447901:0:64013:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:2.0:1775764782.447907:0:64013:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764782.447910:0:64013:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764782.447912:0:64013:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17be41 00000080:00010000:2.0:1775764782.447916:0:64013:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17be41 from it ffffa60b01dabdc8 00010000:00010000:2.0:1775764782.447918:0:64013:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:2.0:1775764782.447925:0:64013:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000080:00200000:2.0:1775764782.447932:0:64013:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:2.0:1775764782.447937:0:64013:0:(namei.c:2226:do_mkdir()) VFS Op:name=d24b.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08) 00800000:00000004:2.0:1775764782.448097:0:64013:0:(obd_class.h:1058:obd_statfs_async()) lustre-MDT0000-mdc-ffff8b2389398000: age 2180, max_age 0 00010000:00010000:2.0:1775764782.448321:0:64013:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:2.0:1775764782.448364:0:64013:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:2.0:1775764782.448370:0:64013:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764782.448372:0:64013:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17be41 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052039 expref: -99 pid: 63179 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000020:00100000:2.0:1775764782.448397:0:64013:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764782.448405:0:64013:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6680 x1862022098634752/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 560/984 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764782.448423:0:64013:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job mkdir:fcf15866-4739-4595-8fe1-44abde7ee9d7:64013:1862022098634752:192.168.202.128@tcp:36:mkdir.0 00000100:00100000:2.0:1775764782.448666:0:64013:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c7c00 going to sleep for 11 seconds 00000100:00100000:1.0:1775764782.454229:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119060864 00000100:00100000:1.0:1775764782.454239:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119060864 00000100:00100000:1.0:1775764782.454293:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 255 00000100:00100000:1.0:1775764782.454303:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:44493:x1862022119060864:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764782.454320:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764782.454397:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764782.454403:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:44493:x1862022119060864:12345-192.168.202.128@tcp:104: Request processed in 102us (454us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764782.454411:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 255 00010000:00010000:2.0:1775764782.454967:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764782.454974:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764782.454988:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8741600 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764782.454993:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764782.455002:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764782.455011:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764782.455061:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 2/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764782.455077:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764782.455101:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764782.455108:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741600/0x73cfd86cfc17beb8 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030521c1 expref: -99 pid: 63726 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764782.455130:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764782.455132:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764782.455134:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764782.468367:0:64013:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job mkdir:fcf15866-4739-4595-8fe1-44abde7ee9d7:64013:1862022098634752:192.168.202.128@tcp:36:mkdir.0 00000080:00200000:2.0:1775764782.468432:0:64013:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88) 00000080:00200000:2.0:1775764782.468439:0:64013:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xd:0x0], updating i_size 512 00000080:00200000:2.0:1775764782.475877:0:64014:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764782.475905:0:64014:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764782.475971:0:64014:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 64014 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.475982:0:64014:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64014 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.475992:0:64014:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64014 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764782.476019:0:64014:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64014:1862022098635008:192.168.202.128@tcp:101:dd.0 00000100:00100000:2.0:1775764782.476166:0:64014:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c7c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764782.478826:0:64014:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64014:1862022098635008:192.168.202.128@tcp:101:dd.0 00010000:00010000:2.0:1775764782.478861:0:64014:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.478873:0:64014:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.478886:0:64014:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.478897:0:64014:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.478909:0:64014:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.478918:0:64014:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.478925:0:64014:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764782.478936:0:64014:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea7480 x1862022098635008/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764798 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764782.478949:0:64014:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.478960:0:64014:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764782.478967:0:64014:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764782.478995:0:64014:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764782.479000:0:64014:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:2.0:1775764782.479005:0:64014:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764782.479008:0:64014:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01e7bbf8 00010000:00010000:2.0:1775764782.479011:0:64014:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.479019:0:64014:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764782.479043:0:64014:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764782.479060:0:64014:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=d24b.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), flags=25 00000080:00200000:2.0:1775764782.479076:0:64014:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=d24b.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=0 00000002:00010000:2.0:1775764782.479098:0:64014:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d24b.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:2.0:1775764782.479135:0:64014:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17becd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64014 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.479143:0:64014:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17becd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64014 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.479152:0:64014:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17becd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64014 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764782.479172:0:64014:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64014:1862022098635136:192.168.202.128@tcp:101:dd.0 00000100:00100000:2.0:1775764782.479239:0:64014:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c7c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764782.487115:0:64014:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64014:1862022098635136:192.168.202.128@tcp:101:dd.0 00010000:00010000:2.0:1775764782.487139:0:64014:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17becd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305221c expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.487160:0:64014:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17becd lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305221c expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.487169:0:64014:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17becd lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305221c expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.487180:0:64014:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17becd lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305221c expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.487190:0:64014:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17becd lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305221c expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.487201:0:64014:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17becd lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305221c expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.487214:0:64014:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17becd lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305221c expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.487222:0:64014:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17becd lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305221c expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764782.487232:0:64014:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=b, status=0 req@ffff8b23b9ea7480 x1862022098635136/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764798 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764782.487244:0:64014:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17becd lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305221c expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.487254:0:64014:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/13 (0 0) 00000080:00200000:2.0:1775764782.487260:0:64014:0:(namei.c:953:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764782.487281:0:64014:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xd:0x0], updating i_size 512 00000080:00200000:2.0:1775764782.487284:0:64014:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0xd:0x0](ffff8b2390ec3a88): rc = 0 00000080:00010000:2.0:1775764782.487289:0:64014:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xd:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17becd 00000080:00010000:2.0:1775764782.487301:0:64014:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xd:0x0](ffff8b2390ec3a88) 00000080:00010000:2.0:1775764782.487304:0:64014:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17becd from it ffffa60b01e7bac8 00010000:00010000:2.0:1775764782.487308:0:64014:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17becd lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305221c expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.487316:0:64014:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17becd lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305221c expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764782.487344:0:64014:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:2.0:1775764782.487362:0:64014:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), inode mode 41ed mask 3 00000080:00200000:2.0:1775764782.487368:0:64014:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f24b.sanity-pcc, dir=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), file ffff8b2388011500, open_flags 8241, mode 81b6 opened 0 00000080:00200000:2.0:1775764782.487384:0:64014:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f24b.sanity-pcc, dir=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), intent=open|creat 00000002:00010000:2.0:1775764782.488204:0:64014:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f24b.sanity-pcc,[0x200003ab4:0xe:0x0]) in obj [0x200003ab4:0xd:0x0], intent: open|creat flags 0101102 00010000:00010000:2.0:1775764782.488264:0:64014:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 3/0,1 mode: --/CW res: [0x200003ab4:0xd:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64014 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.488274:0:64014:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 3/0,1 mode: --/CW res: [0x200003ab4:0xd:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64014 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764782.488285:0:64014:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764782.489236:0:64014:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7480 x1862022098635392/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764782.489248:0:64014:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 3/0,1 mode: --/CW res: [0x200003ab4:0xd:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64014 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764782.489269:0:64014:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64014:1862022098635392:192.168.202.128@tcp:101:dd.0 00000100:00100000:2.0:1775764782.489465:0:64014:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c7c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764782.501275:0:64014:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64014:1862022098635392:192.168.202.128@tcp:101:dd.0 00010000:00010000:2.0:1775764782.501302:0:64014:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 4/0,1 mode: --/CW res: [0x200003ab4:0xd:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.501322:0:64014:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.501330:0:64014:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.501342:0:64014:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.501353:0:64014:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.501366:0:64014:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.501374:0:64014:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.501381:0:64014:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764782.501391:0:64014:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8b23b9ea7480 x1862022098635392/t34359739207(34359739207) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764798 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764782.501403:0:64014:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7480 x1862022098635392/t34359739207(34359739207) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764798 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764782.501434:0:64014:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764782.501446:0:64014:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.501492:0:64014:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/14 (0 0) 00000080:00200000:2.0:1775764782.501507:0:64014:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200003ab4:0xd:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764782.501522:0:64014:0:(namei.c:1620:ll_create_it()) VFS Op:name=f24b.sanity-pcc, dir=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), intent=open|creat 00000080:00200000:2.0:1775764782.501781:0:64014:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab4:0xe:0x0](ffff8b2390ec4b08) 00000080:00200000:2.0:1775764782.501897:0:64014:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0xe:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1775764782.501907:0:64014:0:(namei.c:1586:ll_create_node()) setting l_ast_data to inode [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) 00000080:00010000:2.0:1775764782.501911:0:64014:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) for lock 0x73cfd86cfc17bed4 00000080:00010000:2.0:1775764782.501920:0:64014:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) 00000080:00200008:2.0:1775764782.501934:0:64014:0:(file.c:1057:ll_file_open()) START file f24b.sanity-pcc:[0x200003ab4:0xe:0x0](ffff8b2390ec4b08), flags 101101 00000002:00100000:2.0:1775764782.501952:0:64014:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7480 x1862022098635392/t34359739207(34359739207) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764798 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764782.501989:0:64014:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bed4 from it ffff8b238504c000 00010000:00010000:2.0:1775764782.501993:0:64014:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.502003:0:64014:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764782.502042:0:64014:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xe:0x0](ffff8b2390ec4b08), inode mode 81a4 mask 40 00000080:00200008:2.0:1775764782.502145:0:64014:0:(file.c:2612:do_file_write_iter()) START file f24b.sanity-pcc:[0x200003ab4:0xe:0x0], ppos: 0, count: 1024 00000080:00200000:2.0:1775764782.502158:0:64014:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 1024 bytes 00000080:00200000:2.0:1775764782.502164:0:64014:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 1024 00000080:00200000:2.0:1775764782.502176:0:64014:0:(file.c:2058:ll_file_io_generic()) f24b.sanity-pcc: write ppos: 0, bytes: 1024 00000080:00200000:2.0:1775764782.502180:0:64014:0:(file.c:1815:ll_io_set_mirror()) f24b.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764782.502187:0:64014:0:(cl_io.c:214:cl_io_rw_init()) header@00000000c8206275[0x110, 1, [0x200003ab4:0xe:0x0] hash] 00000020:00200000:2.0:1775764782.502189:0:64014:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 1024) 0 0 00000080:00200000:2.0:1775764782.502194:0:64014:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0xe:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764782.502206:0:64014:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:2.0:1775764782.502214:0:64014:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1024) 1024, 1024 00020000:00200000:2.0:1775764782.502219:0:64014:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764782.502243:0:64014:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1775764782.502257:0:64014:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764782.502262:0:64014:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8430, osc lock ffff8b238a3f8850, flags 0x0 00010000:00010000:2.0:1775764782.502306:0:64014:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 3/0,1 mode: --/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.502316:0:64014:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 3/0,1 mode: --/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.502326:0:64014:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 3/0,1 mode: --/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764782.502345:0:64014:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64014:1862022098635520:192.168.202.128@tcp:101:dd.0 00000100:00100000:2.0:1775764782.502916:0:64014:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c7900 going to sleep for 11 seconds 00000100:00100000:2.0:1775764782.508857:0:64014:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64014:1862022098635520:192.168.202.128@tcp:101:dd.0 00010000:00010000:2.0:1775764782.508879:0:64014:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 4/0,1 mode: --/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.508893:0:64014:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.508901:0:64014:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.508911:0:64014:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1775764782.508920:0:64014:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.508928:0:64014:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.508934:0:64014:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764782.508955:0:64014:0:(vvp_io.c:1286:vvp_io_write_start()) f24b.sanity-pcc: write [0, 1024) append 0 append lockpos 0 00000080:00200000:2.0:1775764782.508969:0:64014:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 1024 bytes 00000080:00200000:2.0:1775764782.509014:0:64014:0:(rw26.c:858:ll_write_end()) pos 0, len 1024, copied 1024 00000080:00200000:2.0:1775764782.509021:0:64014:0:(rw26.c:916:ll_write_end()) page@00000000610a5a6c[3 000000004e2a4a08 2 1 0000000000567e49] 00000080:00200000:2.0:1775764782.509024:0:64014:0:(rw26.c:916:ll_write_end()) vmpage @0000000056f5d604 17ffffc0001001 3:0 ffff8b23baeaeee0 0 lru 00000080:00200000:2.0:1775764782.509029:0:64014:0:(rw26.c:916:ll_write_end()) osc-page@0000000075db91a1 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 000000003596ffcd 00000000f82d4cde > 3< 0 0 > 4< 0 0 8 13631488 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764782.509030:0:64014:0:(rw26.c:916:ll_write_end()) end page@00000000610a5a6c 00000080:00200000:2.0:1775764782.509031:0:64014:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764782.509035:0:64014:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 1024 prio 0 00000008:00010000:2.0:1775764782.509052:0:64014:0:(osc_cache.c:848:osc_extent_find()) ### extent: 00000000fa56d242 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764782.509082:0:64014:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2390ec4cb8, count 1, dirtied 1 00000080:00200000:2.0:1775764782.509088:0:64014:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 1024 bytes, tot: 1024 00000080:00200000:2.0:1775764782.509091:0:64014:0:(vvp_io.c:1353:vvp_io_write_start()) f24b.sanity-pcc: write bytes 0, result: 1024 00000020:00010000:2.0:1775764782.509112:0:64014:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764782.509116:0:64014:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764782.509119:0:64014:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8430 (ffff8b23812ec5a8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764782.509124:0:64014:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.509133:0:64014:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1775764782.509153:0:64014:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764782.509156:0:64014:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8430 (ffff8b23812eca28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764782.509163:0:64014:0:(vvp_io.c:517:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:2.0:1775764782.509169:0:64014:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:2.0:1775764782.509208:0:64014:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xe:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764782.509212:0:64014:0:(file.c:2206:ll_file_io_generic()) f24b.sanity-pcc: 2 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:2.0:1775764782.509220:0:64014:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0xe:0x0] updating i_size 1024 i_blocks 1 00000080:00200000:2.0:1775764782.509225:0:64014:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 1024 00000080:00200008:2.0:1775764782.509271:0:64014:0:(file.c:447:ll_file_release()) START file f24b.sanity-pcc:[0x200003ab4:0xe:0x0](ffff8b2390ec4b08), flags 100001 00000002:00100000:2.0:1775764782.509324:0:64014:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea7480 x1862022098635392/t34359739207(34359739207) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764798 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764782.509334:0:64014:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764782.509339:0:64014:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4a80 x1862022098635776/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764782.509360:0:64014:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64014:1862022098635776:192.168.202.128@tcp:35:dd.0 00000100:00100000:2.0:1775764782.509879:0:64014:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c7900 going to sleep for 11 seconds 00000100:00100000:2.0:1775764782.513108:0:64014:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64014:1862022098635776:192.168.202.128@tcp:35:dd.0 00000002:00100000:2.0:1775764782.513134:0:64014:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea7480 x1862022098635392/t34359739207(34359739207) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764798 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764782.523170:0:64015:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764782.523197:0:64015:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764782.523215:0:64015:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764782.523240:0:64015:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764782.523249:0:64015:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764782.523257:0:64015:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764782.523261:0:64015:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:2.0:1775764782.523268:0:64015:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01ebbb58 00010000:00010000:2.0:1775764782.523271:0:64015:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764782.523280:0:64015:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764782.523310:0:64015:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764782.523318:0:64015:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24b.sanity-pcc, flags=25 00000080:00200000:2.0:1775764782.523324:0:64015:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:2.0:1775764782.523328:0:64015:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f24b.sanity-pcc, flags=9 00000080:00200008:2.0:1775764782.523334:0:64015:0:(file.c:6226:ll_getattr_dentry()) START file f24b.sanity-pcc:[0x200003ab4:0xe:0x0](ffff8b2390ec4b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764782.523338:0:64015:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0xe:0x0](ffff8b2390ec4b08),name=f24b.sanity-pcc 00000002:00010000:2.0:1775764782.523349:0:64015:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xe:0x0]) in obj [0x200003ab4:0xd:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764782.523357:0:64015:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764782.523365:0:64015:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764782.523372:0:64015:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200003ab4:0xd:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764782.523376:0:64015:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) 00000080:00010000:2.0:1775764782.523379:0:64015:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) for lock 0x73cfd86cfc17bed4 00000080:00010000:2.0:1775764782.523383:0:64015:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bed4 from it ffffa60b01ebbcb8 00010000:00010000:2.0:1775764782.523386:0:64015:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764782.523393:0:64015:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764782.523579:0:64015:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0xe:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1775764782.523589:0:64015:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0xe:0x0] 00000008:00010000:2.0:1775764782.523618:0:64015:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764782.523622:0:64015:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8430, osc lock ffff8b238a3f8850, flags 0x400001002 00010000:00010000:2.0:1775764782.523642:0:64015:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764782.523654:0:64015:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1775764782.523665:0:64015:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764782.523669:0:64015:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764782.523677:0:64015:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 5/0,0 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1775764782.523688:0:64015:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764782.523698:0:64015:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0xe:0x0] updating i_size 1024 i_blocks 1 00000020:00010000:2.0:1775764782.523702:0:64015:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764782.523705:0:64015:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764782.523708:0:64015:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8430 (ffff8b23812ec5a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764782.523712:0:64015:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764782.523714:0:64015:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8430 (ffff8b23812eca28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764782.523733:0:64015:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xe:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764782.523760:0:64015:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764782.523767:0:64015:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764782.523773:0:64015:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764782.523782:0:64015:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764782.523788:0:64015:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764782.523792:0:64015:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764782.523794:0:64015:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:2.0:1775764782.523798:0:64015:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01ebbc08 00010000:00010000:2.0:1775764782.523800:0:64015:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764782.523807:0:64015:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764782.523816:0:64015:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764782.523820:0:64015:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24b.sanity-pcc, flags=25 00000080:00200000:2.0:1775764782.523823:0:64015:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:2.0:1775764782.523826:0:64015:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f24b.sanity-pcc, flags=9 00000080:00200008:2.0:1775764782.523839:0:64015:0:(llite_lib.c:2274:ll_setattr_raw()) START file f24b.sanity-pcc:[0x200003ab4:0xe:0x0](ffff8b2390ec4b08) current size 1024, valid attrs 41, mode 8180, uid -1604206592, gid -29917, new size 0, atime 1775764782.000000000, mtime 1775764782.000000000, ctime 1775764782.000000000 00010000:00010000:2.0:1775764782.523856:0:64015:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764782.523880:0:64015:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0xe:0x0]: losing layout lock 00010000:00010000:2.0:1775764782.523918:0:64015:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764782.523923:0:64015:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764782.523925:0:64015:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bed4 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052238 expref: -99 pid: 64014 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000020:00100000:2.0:1775764782.523941:0:64015:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764782.523952:0:64015:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5c00 x1862022098636032/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764782.523975:0:64015:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job chmod:fcf15866-4739-4595-8fe1-44abde7ee9d7:64015:1862022098636032:192.168.202.128@tcp:36:chmod.0 00000100:00100000:2.0:1775764782.524076:0:64015:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c7900 going to sleep for 11 seconds 00000100:00100000:2.0:1775764782.527994:0:64015:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job chmod:fcf15866-4739-4595-8fe1-44abde7ee9d7:64015:1862022098636032:192.168.202.128@tcp:36:chmod.0 00000080:00010000:2.0:1775764782.528045:0:64015:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) 00000002:00010000:2.0:1775764782.528055:0:64015:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xe:0x0]) in obj [0x200003ab4:0xe:0x0], intent: layout flags 0 00010000:00010000:2.0:1775764782.528112:0:64015:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bee2 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 64015 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.528122:0:64015:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bee2 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64015 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.528133:0:64015:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bee2 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64015 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764782.528153:0:64015:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job chmod:fcf15866-4739-4595-8fe1-44abde7ee9d7:64015:1862022098636160:192.168.202.128@tcp:101:chmod.0 00000100:00100000:2.0:1775764782.529081:0:64015:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c7900 going to sleep for 11 seconds 00000080:00200000:2.0:1775764782.543930:0:64015:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xe:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1775764782.543954:0:64015:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file f24b.sanity-pcc:[0x200003ab4:0xe:0x0](ffff8b2390ec4b08) current size 1024, valid attrs 41, mode 8180, uid -1604206592, gid -29917, new size 0, atime 1775764782.000000000, mtime 1775764782.000000000, ctime 1775764782.000000000, rc 0 00000080:00200000:1.0:1775764782.608299:0:64021:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764782.608325:0:64021:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764782.608343:0:64021:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764782.608357:0:64021:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764782.608364:0:64021:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:1.0:1775764782.608371:0:64021:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764782.608375:0:64021:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:1.0:1775764782.608381:0:64021:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01edbbf8 00010000:00010000:1.0:1775764782.608385:0:64021:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764782.608394:0:64021:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764782.608410:0:64021:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000100:00100000:1.0:1775764782.608490:0:64021:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64021:1862022098636416:192.168.202.128@tcp:49:dd.500 00000100:00100000:1.0:1775764782.608920:0:64021:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764782.612849:0:64021:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64021:1862022098636416:192.168.202.128@tcp:49:dd.500 00000080:00200000:2.0:1775764782.612885:0:64021:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24b.sanity-pcc, flags=25 00000080:00200000:2.0:1775764782.612894:0:64021:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000100:00100000:2.0:1775764782.612941:0:64021:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64021:1862022098636544:192.168.202.128@tcp:49:dd.500 00000100:00100000:2.0:1775764782.613017:0:64021:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c70c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764782.615918:0:64021:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64021:1862022098636544:192.168.202.128@tcp:49:dd.500 00000080:00200000:2.0:1775764782.616033:0:64021:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), inode mode 41ed mask 3 00000080:00200000:2.0:1775764782.616042:0:64021:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f24b.sanity-pcc, dir=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), file ffff8b2389073500, open_flags 8201, mode 81b6 opened 0 00000080:00200000:2.0:1775764782.616049:0:64021:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f24b.sanity-pcc, dir=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), intent=open 00000002:00010000:2.0:1775764782.616080:0:64021:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f24b.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200003ab4:0xd:0x0], intent: open flags 0101002 00010000:00010000:2.0:1775764782.616139:0:64021:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bee9 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64021 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.616149:0:64021:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bee9 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64021 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764782.616159:0:64021:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764782.616166:0:64021:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5500 x1862022098636800/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764782.616181:0:64021:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bee9 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64021 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764782.616200:0:64021:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64021:1862022098636800:192.168.202.128@tcp:101:dd.500 00000100:00100000:2.0:1775764782.616386:0:64021:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c70c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764782.624836:0:64021:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64021:1862022098636800:192.168.202.128@tcp:101:dd.500 00010000:00010000:2.0:1775764782.624859:0:64021:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bee9 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64021 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.624871:0:64021:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bee9 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64021 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.624878:0:64021:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bee9 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64021 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.624885:0:64021:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bee9 lrc: 3/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64021 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.624892:0:64021:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bee9 lrc: 4/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64021 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.624898:0:64021:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6ebc00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764782.624902:0:64021:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bee9 lrc: 4/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64021 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.624909:0:64021:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bee9 lrc: 4/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64021 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.624916:0:64021:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bee9 lrc: 5/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 64021 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.624923:0:64021:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764782.624928:0:64021:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bee9 lrc: 3/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64021 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.624934:0:64021:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bee9 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64021 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764782.625006:0:64021:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=1000000b, status=-13 req@ffff8b23b9ea5500 x1862022098636800/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764798 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'dd.500' uid:500 gid:500 projid:0 00000080:00200000:2.0:1775764782.625021:0:64021:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab4:0xd:0x0] suppgids -1 -1: rc -13 00000100:00100000:2.0:1775764782.703164:0:64027:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf800 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64027:1862022098637056:192.168.202.128@tcp:101:dd.500 00010000:00010000:2.0:1775764782.703190:0:64027:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bef0 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.703875:0:64027:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bef0 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.703883:0:64027:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bef0 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.703890:0:64027:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bef0 lrc: 3/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.703899:0:64027:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bef0 lrc: 4/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.703905:0:64027:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23c2045600 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764782.703910:0:64027:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bef0 lrc: 4/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.703917:0:64027:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bef0 lrc: 4/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.703925:0:64027:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bef0 lrc: 5/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 64027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.703932:0:64027:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764782.703937:0:64027:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bef0 lrc: 3/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64027 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.703944:0:64027:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bef0 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64027 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764782.704035:0:64027:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=1000000b, status=-13 req@ffff8b2382daf800 x1862022098637056/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764798 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'dd.500' uid:500 gid:500 projid:0 00000080:00200000:2.0:1775764782.704049:0:64027:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab4:0xd:0x0] suppgids -1 -1: rc -13 00000100:00100000:1.0:1775764782.781399:0:64033:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64033:1862022098637184:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764782.781429:0:64033:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bef7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305229a expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.781446:0:64033:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bef7 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305229a expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.781531:0:64033:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bef7 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305229a expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.781543:0:64033:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bef7 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305229a expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.781789:0:64033:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bef7 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305229a expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.781801:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bef7 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305229a expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.781812:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bef7 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305229a expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.781820:0:64033:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bef7 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305229a expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764782.781831:0:64033:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b2382dac380 x1862022098637184/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764798 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764782.781843:0:64033:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bef7 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305229a expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.781853:0:64033:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/13 (0 0) 00000080:00200000:1.0:1775764782.781858:0:64033:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764782.781878:0:64033:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xd:0x0], updating i_size 512 00000080:00010000:1.0:1775764782.781884:0:64033:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xd:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17bef7 00000080:00010000:1.0:1775764782.781893:0:64033:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xd:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764782.781896:0:64033:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bef7 from it ffffa60b01dcbcb8 00010000:00010000:1.0:1775764782.781899:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bef7 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305229a expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.781907:0:64033:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bef7 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305229a expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764782.781976:0:64033:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764782.781991:0:64033:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764782.782001:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764782.782012:0:64033:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764782.782018:0:64033:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764782.782023:0:64033:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764782.782025:0:64033:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:1.0:1775764782.782028:0:64033:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01dcbb58 00010000:00010000:1.0:1775764782.782031:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764782.782039:0:64033:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764782.782047:0:64033:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764782.782053:0:64033:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24b.sanity-pcc, flags=24 00000080:00200000:1.0:1775764782.782057:0:64033:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764782.782078:0:64033:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=f24b.sanity-pcc, dir=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), flags=8 00000080:00200000:1.0:1775764782.782082:0:64033:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f24b.sanity-pcc, dir=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), intent=getattr 00000002:00010000:1.0:1775764782.782095:0:64033:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f24b.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200003ab4:0xd:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764782.782132:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 64033 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.782140:0:64033:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 64033 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.782150:0:64033:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 64033 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764782.782175:0:64033:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64033:1862022098637312:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764782.782300:0:64033:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6fcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764782.787113:0:64033:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64033:1862022098637312:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764782.787137:0:64033:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.787155:0:64033:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.787163:0:64033:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.787174:0:64033:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.787185:0:64033:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.787198:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.787206:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.787213:0:64033:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764782.787228:0:64033:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864fca80 x1862022098637312/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/640 e 0 to 0 dl 1775764798 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764782.787238:0:64033:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764782.787250:0:64033:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.787260:0:64033:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/14 (0 0) 00000080:00200000:1.0:1775764782.787265:0:64033:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200003ab4:0xd:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764782.787346:0:64033:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0xe:0x0](ffff8b2390ec4b08): rc = 0 00000080:00200000:1.0:1775764782.787363:0:64033:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0xe:0x0]: layout version change: 0 -> 0 00000080:00010000:1.0:1775764782.788114:0:64033:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) for lock 0x73cfd86cfc17befe 00000080:00010000:1.0:1775764782.788131:0:64033:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) 00000080:00010000:1.0:1775764782.788134:0:64033:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17befe from it ffffa60b01dcbb20 00010000:00010000:1.0:1775764782.788139:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.788149:0:64033:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:1.0:1775764782.788188:0:64033:0:(file.c:6226:ll_getattr_dentry()) START file f24b.sanity-pcc:[0x200003ab4:0xe:0x0](ffff8b2390ec4b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764782.788193:0:64033:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0xe:0x0](ffff8b2390ec4b08),name=f24b.sanity-pcc 00000002:00010000:1.0:1775764782.788211:0:64033:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xe:0x0]) in obj [0x200003ab4:0xd:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764782.788221:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764782.788237:0:64033:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764782.788244:0:64033:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200003ab4:0xd:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764782.788247:0:64033:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) 00000080:00010000:1.0:1775764782.788250:0:64033:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) for lock 0x73cfd86cfc17befe 00000080:00010000:1.0:1775764782.788253:0:64033:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17befe from it ffffa60b01dcbcb8 00010000:00010000:1.0:1775764782.788256:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764782.788263:0:64033:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764782.788303:0:64033:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0xe:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764782.788311:0:64033:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0xe:0x0] 00000008:00010000:1.0:1775764782.788333:0:64033:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764782.788335:0:64033:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5930, osc lock ffff8b2382cf6130, flags 0x400001002 00010000:00010000:1.0:1775764782.788350:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764782.788361:0:64033:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1775764782.788682:0:64033:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764782.788687:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764782.788703:0:64033:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 5/0,0 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1775764782.788713:0:64033:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764782.788724:0:64033:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0xe:0x0] updating i_size 1024 i_blocks 1 00000020:00010000:1.0:1775764782.788727:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_release():245 00000020:00010000:1.0:1775764782.788730:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764782.788733:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5930 (ffff8b2387cba008/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764782.788736:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764782.788739:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5930 (ffff8b2387cba368/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764782.788757:0:64033:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xe:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764782.788803:0:64033:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764782.788811:0:64033:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764782.788819:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764782.788828:0:64033:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764782.788835:0:64033:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764782.788839:0:64033:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764782.788841:0:64033:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:1.0:1775764782.788845:0:64033:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01dcbbf8 00010000:00010000:1.0:1775764782.788848:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764782.789087:0:64033:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764782.789098:0:64033:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764782.789105:0:64033:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24b.sanity-pcc, flags=25 00000080:00200000:1.0:1775764782.789110:0:64033:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764782.789113:0:64033:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f24b.sanity-pcc, flags=265 00000080:00200000:1.0:1775764782.789117:0:64033:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xe:0x0](ffff8b2390ec4b08), inode mode 8180 mask 44 00000080:00200008:1.0:1775764782.789124:0:64033:0:(file.c:1057:ll_file_open()) START file f24b.sanity-pcc:[0x200003ab4:0xe:0x0](ffff8b2390ec4b08), flags 100000 00000002:00010000:1.0:1775764782.789137:0:64033:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xe:0x0]) in obj [0x200003ab4:0xd:0x0], intent: open flags 0240100001 00010000:00010000:1.0:1775764782.789180:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 64033 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764782.789188:0:64033:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 64033 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764782.789198:0:64033:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764782.789208:0:64033:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fca80 x1862022098637568/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764782.789218:0:64033:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 64033 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764782.789236:0:64033:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64033:1862022098637568:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764782.789921:0:64033:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6fcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764782.794111:0:64033:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64033:1862022098637568:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764782.794136:0:64033:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522c4 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.794152:0:64033:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522c4 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.794161:0:64033:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522c4 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.794173:0:64033:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030522c4 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.794189:0:64033:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522c4 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.794202:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030522c4 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.794210:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030522c4 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.794217:0:64033:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030522c4 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764782.794227:0:64033:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23864fca80 x1862022098637568/t34359739210(34359739210) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764798 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764782.794240:0:64033:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fca80 x1862022098637568/t34359739210(34359739210) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764798 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764782.794251:0:64033:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522c4 expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764782.794263:0:64033:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522c4 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.794274:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764782.794286:0:64033:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764782.794294:0:64033:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522c4 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.794300:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e030522c4 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.794306:0:64033:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e030522c4 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.794329:0:64033:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:2.0:1775764782.794336:0:64033:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab4:0xd:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764782.794439:0:64033:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0xe:0x0]: layout version change: 0 -> 0 00000080:00010000:2.0:1775764782.794449:0:64033:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) for lock 0x73cfd86cfc17befe 00000080:00010000:2.0:1775764782.794479:0:64033:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17befe from it ffffa60b01dcbbf0 00010000:00010000:2.0:1775764782.794483:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764782.794492:0:64033:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000002:00100000:2.0:1775764782.794514:0:64033:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fca80 x1862022098637568/t34359739210(34359739210) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764798 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764782.794691:0:64033:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0xe:0x0](ffff8b2390ec4b08) cmd=400866fc arg=7ffc5d1afa08 00000080:00010000:2.0:1775764782.794726:0:64033:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) 00000002:00010000:2.0:1775764782.794735:0:64033:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xe:0x0]) in obj [0x200003ab4:0xe:0x0], intent: layout flags 02 00010000:00010000:2.0:1775764782.794750:0:64033:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e030522c4 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764782.794771:0:64033:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0xe:0x0]: losing layout lock 00010000:00010000:2.0:1775764782.794789:0:64033:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764782.794804:0:64033:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0xe:0x0]: losing layout lock 00010000:00010000:2.0:1775764782.794814:0:64033:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bee2 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052254 expref: -99 pid: 64015 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764782.794828:0:64033:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0xe:0x0]: losing layout lock 00010000:00010000:2.0:1775764782.794855:0:64033:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf05 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030522c4 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.794862:0:64033:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764782.794868:0:64033:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bee2 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052254 expref: -99 pid: 64015 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764782.794873:0:64033:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775764782.794876:0:64033:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17befe lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030522b6 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764782.794888:0:64033:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bee2 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052254 expref: -99 pid: 64015 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764782.794906:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0xe:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64033 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.794912:0:64033:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0xe:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64033 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764782.794923:0:64033:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764782.794929:0:64033:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5500 x1862022098637824/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 392/880 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764782.794937:0:64033:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0xe:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64033 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764782.795313:0:64033:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64033:1862022098637824:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764782.795743:0:64033:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c7540 going to sleep for 11 seconds 00000100:00100000:2.0:1775764782.800628:0:64033:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64033:1862022098637824:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764782.800658:0:64033:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 4/0,1 mode: --/EX res: [0x200003ab4:0xe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.800670:0:64033:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 4/0,1 mode: --/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.800682:0:64033:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.800692:0:64033:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.800704:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.800711:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.800723:0:64033:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764782.800734:0:64033:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea5500 x1862022098637824/t34359739211(34359739211) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 392/600 e 0 to 0 dl 1775764798 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764782.800752:0:64033:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764782.800765:0:64033:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.800777:0:64033:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/14 (0 0) 00000080:00200000:2.0:1775764782.800784:0:64033:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0xe:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764782.800793:0:64033:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) for lock 0x73cfd86cfc17bf0c 00000080:00010000:2.0:1775764782.800801:0:64033:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf0c from it ffffa60b01dcbc10 00010000:00010000:2.0:1775764782.800804:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764782.800812:0:64033:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764782.800834:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764782.800845:0:64033:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764782.800857:0:64033:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0xe:0x0](0000000051d190b3) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764782.800944:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764782.801178:0:64033:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764782.801214:0:64033:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0xe:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764782.801229:0:64033:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:2.0:1775764782.801266:0:64033:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 00000000fa56d242 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:2.0:1775764782.801281:0:64033:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000fa56d242 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764782.801329:0:64033:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00100000:2.0:1775764782.801773:0:64033:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea6300 - object 0x240000400:283 offset 0<>1024 00000080:00200000:2.0:1775764782.808956:0:64033:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xe:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764782.809053:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764782.809080:0:64033:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764782.809091:0:64033:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0xe:0x0](0000000051d190b3) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764782.809155:0:64033:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0xe:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764782.809165:0:64033:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764782.809202:0:64033:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xe:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:2.0:1775764782.809220:0:64033:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 1024 blocks 1 [cma]time 1775764782 1775764782 0 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764782.809279:0:64033:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0xe:0x0]: layout version change: 4294967294 -> 2 00010000:00010000:2.0:1775764782.809290:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764782.809299:0:64033:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf0c lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030522e0 expref: -99 pid: 64033 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:2.0:1775764782.809909:0:64033:0:(file.c:2454:do_file_read_iter()) START file f24b.sanity-pcc:[0x200003ab4:0xe:0x0], ppos: 0, count: 1048576 00000080:00200000:2.0:1775764782.809935:0:64033:0:(rw.c:1928:ll_readpage()) fast read pgno: 0 00000080:00200000:2.0:1775764782.809957:0:64033:0:(file.c:2058:ll_file_io_generic()) f24b.sanity-pcc: read ppos: 0, bytes: 1048576 00000080:00200000:2.0:1775764782.809962:0:64033:0:(file.c:1815:ll_io_set_mirror()) f24b.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764782.809969:0:64033:0:(cl_io.c:214:cl_io_rw_init()) header@00000000c8206275[0x110, 1, [0x200003ab4:0xe:0x0] hash] 00000020:00200000:2.0:1775764782.809971:0:64033:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 1048576) 0 0 00000080:00200000:2.0:1775764782.809974:0:64033:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0xe:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764782.809985:0:64033:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0xe:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764782.809989:0:64033:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764782.809994:0:64033:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:2.0:1775764782.809998:0:64033:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764782.810018:0:64033:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:2.0:1775764782.810031:0:64033:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764782.810035:0:64033:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8130, osc lock ffff8b238a3f8850, flags 0x400000000000000 00010000:00010000:2.0:1775764782.810051:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764782.810073:0:64033:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764782.810084:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00010000:2.0:1775764782.810094:0:64033:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 1024, setting rss=1024; kms=1024, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764782.810103:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764782.810112:0:64033:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764782.810121:0:64033:0:(vvp_io.c:826:vvp_io_read_start()) f24b.sanity-pcc: read [0, 1048576) 00000001:00010000:2.0:1775764782.810138:0:64033:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0xe:0x0] 00000008:00010000:2.0:1775764782.810144:0:64033:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764782.810146:0:64033:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c84b0, osc lock ffff8b238a3f8000, flags 0x400000400001002 00010000:00010000:2.0:1775764782.810154:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764782.810163:0:64033:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:2.0:1775764782.810172:0:64033:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764782.810176:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764782.810182:0:64033:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:2.0:1775764782.810188:0:64033:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764782.810194:0:64033:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0xe:0x0] updating i_size 1024 i_blocks 1 00000020:00010000:2.0:1775764782.810198:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764782.810200:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764782.810203:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c84b0 (ffff8b23812ec5a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764782.810207:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764782.810209:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c84b0 (ffff8b23812eca28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764782.810216:0:64033:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:0 00000008:00010000:2.0:1775764782.810254:0:64033:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b2383c9e240 00000008:00010000:2.0:1775764782.810303:0:46654:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b2383c9e240 00010000:00010000:2.0:1775764782.810322:0:46654:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00100000:2.0:1775764782.810365:0:46654:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea5880 - object 0x240000400:283 offset 0<>4096 00000020:00010000:2.0:1775764782.815660:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764782.815666:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764782.815671:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8130 (ffff8b23812ec5a8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764782.815680:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764782.815702:0:64033:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00010000:2.0:1775764782.815718:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764782.815721:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8130 (ffff8b23812eca28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764782.815729:0:64033:0:(vvp_io.c:517:vvp_io_advance()) advancing 1024 bytes 00000080:00200000:2.0:1775764782.815757:0:64033:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xe:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764782.815762:0:64033:0:(file.c:2206:ll_file_io_generic()) f24b.sanity-pcc: 1 io complete with rc: 0, result: 1024, restart: 0 00000080:00200000:2.0:1775764782.815767:0:64033:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 1024 00000080:00200008:2.0:1775764782.815891:0:64033:0:(file.c:2454:do_file_read_iter()) START file f24b.sanity-pcc:[0x200003ab4:0xe:0x0], ppos: 1024, count: 1048576 00000080:00200000:2.0:1775764782.815914:0:64033:0:(file.c:2058:ll_file_io_generic()) f24b.sanity-pcc: read ppos: 1024, bytes: 1048576 00000080:00200000:2.0:1775764782.815919:0:64033:0:(file.c:1815:ll_io_set_mirror()) f24b.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764782.815926:0:64033:0:(cl_io.c:214:cl_io_rw_init()) header@00000000c8206275[0x110, 2, [0x200003ab4:0xe:0x0] hash] 00000020:00200000:2.0:1775764782.815929:0:64033:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [1024, 1049600) 0 0 00000080:00200000:2.0:1775764782.815933:0:64033:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0xe:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764782.816023:0:64033:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0xe:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764782.816027:0:64033:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764782.816035:0:64033:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 1024 chunk: [1024, 1049600) 1049600, 1048576 00020000:00200000:2.0:1775764782.816039:0:64033:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764782.816081:0:64033:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 256] 00000008:00010000:2.0:1775764782.816094:0:64033:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764782.816099:0:64033:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8130, osc lock ffff8b238a3f8850, flags 0x400000000000000 00010000:00010000:2.0:1775764782.816119:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764782.816131:0:64033:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1052671) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764782.816148:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764782.816157:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764782.816164:0:64033:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764782.816173:0:64033:0:(vvp_io.c:826:vvp_io_read_start()) f24b.sanity-pcc: read [1024, 1049600) 00000001:00010000:2.0:1775764782.816188:0:64033:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0xe:0x0] 00000008:00010000:2.0:1775764782.816193:0:64033:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764782.816195:0:64033:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c84b0, osc lock ffff8b238a3f8000, flags 0x400000400001002 00010000:00010000:2.0:1775764782.816203:0:64033:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764782.816212:0:64033:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:2.0:1775764782.816219:0:64033:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764782.816222:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764782.816230:0:64033:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:2.0:1775764782.816242:0:64033:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764782.816248:0:64033:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0xe:0x0] updating i_size 1024 i_blocks 1 00000020:00010000:2.0:1775764782.816250:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764782.816252:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764782.816254:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c84b0 (ffff8b23812ec5a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764782.816257:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764782.816259:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c84b0 (ffff8b23812eca28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764782.816265:0:64033:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:1024 00000020:00010000:2.0:1775764782.816270:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764782.816272:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764782.816274:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8130 (ffff8b23812ec5a8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764782.816277:0:64033:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764782.816285:0:64033:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000085ffb972/0x73cfd86cfc17bedb lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052246 expref: -99 pid: 64014 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000020:00010000:2.0:1775764782.816292:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764782.816294:0:64033:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8130 (ffff8b23812eca28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764782.816298:0:64033:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1775764782.816313:0:64033:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xe:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764782.816316:0:64033:0:(file.c:2206:ll_file_io_generic()) f24b.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1775764782.816318:0:64033:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:2.0:1775764782.816777:0:64033:0:(file.c:447:ll_file_release()) START file f24b.sanity-pcc:[0x200003ab4:0xe:0x0](ffff8b2390ec4b08), flags 100000 00000002:00100000:2.0:1775764782.816833:0:64033:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fca80 x1862022098637568/t34359739210(34359739210) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764798 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764782.816851:0:64033:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764782.816856:0:64033:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4700 x1862022098638336/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764782.816879:0:64033:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64033:1862022098638336:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764782.817373:0:64033:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c7c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764782.820259:0:64033:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64033:1862022098638336:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764782.820294:0:64033:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fca80 x1862022098637568/t34359739210(34359739210) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764798 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764783.031565:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1775764783.050874:0:64049:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764783.050900:0:64049:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764783.050916:0:64049:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764783.050933:0:64049:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764783.050941:0:64049:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:1.0:1775764783.051037:0:64049:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764783.051042:0:64049:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:1.0:1775764783.051049:0:64049:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01efbbf8 00010000:00010000:1.0:1775764783.051053:0:64049:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764783.051214:0:64049:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:1.0:1775764783.051233:0:64049:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764783.051241:0:64049:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24b.sanity-pcc, flags=25 00000080:00200000:1.0:1775764783.051248:0:64049:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000100:00100000:1.0:1775764783.051304:0:64049:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64049:1862022098639104:192.168.202.128@tcp:49:dd.500 00000100:00100000:1.0:1775764783.051688:0:64049:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6f6c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764783.055670:0:64049:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64049:1862022098639104:192.168.202.128@tcp:49:dd.500 00000080:00200000:2.0:1775764783.055717:0:64049:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), inode mode 41ed mask 3 00000080:00200000:2.0:1775764783.055725:0:64049:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f24b.sanity-pcc, dir=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), file ffff8b2383a13b00, open_flags 8201, mode 81b6 opened 0 00000080:00200000:2.0:1775764783.055732:0:64049:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f24b.sanity-pcc, dir=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), intent=open 00000002:00010000:2.0:1775764783.055753:0:64049:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f24b.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200003ab4:0xd:0x0], intent: open flags 0101002 00010000:00010000:2.0:1775764783.055809:0:64049:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf1a lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 64049 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.055819:0:64049:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf1a lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 64049 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764783.055830:0:64049:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764783.055837:0:64049:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4700 x1862022098639360/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764783.055848:0:64049:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf1a lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 64049 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764783.055869:0:64049:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64049:1862022098639360:192.168.202.128@tcp:101:dd.500 00000100:00100000:2.0:1775764783.056238:0:64049:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c7c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764783.061008:0:64049:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64049:1862022098639360:192.168.202.128@tcp:101:dd.500 00010000:00010000:2.0:1775764783.061029:0:64049:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf1a lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 64049 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.061042:0:64049:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf1a lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 64049 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.061049:0:64049:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf1a lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 64049 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.061056:0:64049:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf1a lrc: 3/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 64049 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.061077:0:64049:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf1a lrc: 4/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 64049 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.061083:0:64049:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6eac00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764783.061088:0:64049:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf1a lrc: 4/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 64049 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.061095:0:64049:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf1a lrc: 4/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 64049 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.061102:0:64049:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf1a lrc: 5/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 64049 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.061108:0:64049:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764783.061114:0:64049:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf1a lrc: 3/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 64049 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.061121:0:64049:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf1a lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 64049 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764783.061136:0:64049:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=1000000b, status=-13 req@ffff8b23b9ea4700 x1862022098639360/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/864 e 0 to 0 dl 1775764799 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'dd.500' uid:500 gid:500 projid:0 00000080:00200000:2.0:1775764783.061152:0:64049:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab4:0xd:0x0] suppgids -1 -1: rc -13 00000100:00100000:1.0:1775764783.187411:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098640256:192.168.202.128@tcp:2:chmod.0 00000080:00200000:2.0:1775764783.193631:0:64056:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xe:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1775764783.193651:0:64056:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file f24b.sanity-pcc:[0x200003ab4:0xe:0x0](ffff8b2390ec4b08) current size 1024, valid attrs 41, mode 81ff, uid -2126819328, gid -29917, new size 0, atime 1775764783.000000000, mtime 1775764783.000000000, ctime 1775764783.000000000, rc 0 00000100:00100000:2.0:1775764783.517003:0:64079:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23908dd780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764783.519944:0:64079:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job chown:fcf15866-4739-4595-8fe1-44abde7ee9d7:64079:1862022098642560:192.168.202.128@tcp:36:chown.0 00000080:00200000:2.0:1775764783.520027:0:64079:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0xe:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764783.520040:0:64079:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0xe:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764783.520043:0:64079:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764783.520049:0:64079:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764783.525139:0:64079:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xe:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1775764783.525159:0:64079:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file f24b.sanity-pcc:[0x200003ab4:0xe:0x0](ffff8b2390ec4b08) current size 1024, valid attrs 1842, mode ffff, uid 500, gid -29917, new size 18446615583487369216, atime 1775764783.000000000, mtime 1775764783.000000000, ctime 1775764783.000000000, rc 0 00000080:00200008:1.0:1775764783.611702:0:64085:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764783.611717:0:64085:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764783.611740:0:64085:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764783.611760:0:64085:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:1.0:1775764783.611784:0:64085:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:1.0:1775764783.611791:0:64085:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:1.0:1775764783.611815:0:64085:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764783.611820:0:64085:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:1.0:1775764783.611826:0:64085:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01fafcb8 00010000:00010000:1.0:1775764783.611829:0:64085:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:1.0:1775764783.611836:0:64085:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:1.0:1775764783.611861:0:64085:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764783.611865:0:64085:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764783.611870:0:64085:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:1.0:1775764783.611875:0:64085:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:1.0:1775764783.611879:0:64085:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:1.0:1775764783.611881:0:64085:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764783.611883:0:64085:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:1.0:1775764783.611885:0:64085:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01fafb58 00010000:00010000:1.0:1775764783.611886:0:64085:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:1.0:1775764783.611892:0:64085:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:1.0:1775764783.611896:0:64085:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764783.611901:0:64085:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24b.sanity-pcc, flags=8 00000080:00200008:1.0:1775764783.611906:0:64085:0:(file.c:6226:ll_getattr_dentry()) START file d24b.sanity-pcc:[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764783.611908:0:64085:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88),name=d24b.sanity-pcc 00000002:00010000:1.0:1775764783.611915:0:64085:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xd:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764783.611920:0:64085:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bef7 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305229a expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764783.611925:0:64085:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bef7 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305229a expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764783.611928:0:64085:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:1.0:1775764783.611930:0:64085:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xd:0x0](ffff8b2390ec3a88) 00000080:00010000:1.0:1775764783.611931:0:64085:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xd:0x0](ffff8b2390ec3a88) for lock 0x73cfd86cfc17bef7 00000080:00010000:1.0:1775764783.611933:0:64085:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bef7 from it ffffa60b01fafcb8 00010000:00010000:1.0:1775764783.611934:0:64085:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bef7 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305229a expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764783.611938:0:64085:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bef7 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305229a expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764783.611950:0:64085:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764783.611955:0:64085:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764783.611960:0:64085:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764783.611967:0:64085:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:1.0:1775764783.611973:0:64085:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:1.0:1775764783.611977:0:64085:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764783.611979:0:64085:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:1.0:1775764783.611983:0:64085:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01fafb58 00010000:00010000:1.0:1775764783.611985:0:64085:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1775764783.611992:0:64085:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:1.0:1775764783.612001:0:64085:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764783.612005:0:64085:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24b.sanity-pcc, flags=24 00000080:00200000:1.0:1775764783.612008:0:64085:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:1.0:1775764783.612018:0:64085:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=f24b.sanity-pcc, dir=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), flags=8 00000080:00200000:1.0:1775764783.612022:0:64085:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f24b.sanity-pcc, dir=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), intent=getattr 00000002:00010000:1.0:1775764783.612032:0:64085:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f24b.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200003ab4:0xd:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764783.612095:0:64085:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 64085 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764783.612103:0:64085:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 64085 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764783.612114:0:64085:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 64085 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764783.612146:0:64085:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64085:1862022098642816:192.168.202.128@tcp:101:lfs.500 00000100:00100000:1.0:1775764783.612243:0:64085:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6f6c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764783.630093:0:64085:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64085:1862022098642816:192.168.202.128@tcp:101:lfs.500 00010000:00010000:2.0:1775764783.630117:0:64085:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.630134:0:64085:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.630144:0:64085:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.630155:0:64085:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.630166:0:64085:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.630179:0:64085:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.630187:0:64085:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.630194:0:64085:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764783.630206:0:64085:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864fd180 x1862022098642816/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/840 e 0 to 0 dl 1775764799 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.500' uid:500 gid:500 projid:0 00000002:00010000:2.0:1775764783.630218:0:64085:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764783.630231:0:64085:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.630241:0:64085:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/14 (0 0) 00000080:00200000:2.0:1775764783.630248:0:64085:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200003ab4:0xd:0x0] suppgids -1 -1: rc 0 00000080:00200000:2.0:1775764783.630322:0:64085:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0xe:0x0](ffff8b2390ec4b08): rc = 0 00000080:00200000:2.0:1775764783.630335:0:64085:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0xe:0x0]: layout version change: 2 -> 2 00000080:00010000:2.0:1775764783.630341:0:64085:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) for lock 0x73cfd86cfc17bf52 00000080:00010000:2.0:1775764783.630350:0:64085:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) 00000080:00010000:2.0:1775764783.630352:0:64085:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf52 from it ffffa60b01fafb20 00010000:00010000:2.0:1775764783.630354:0:64085:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.630361:0:64085:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1775764783.630389:0:64085:0:(file.c:6226:ll_getattr_dentry()) START file f24b.sanity-pcc:[0x200003ab4:0xe:0x0](ffff8b2390ec4b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764783.630394:0:64085:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0xe:0x0](ffff8b2390ec4b08),name=f24b.sanity-pcc 00000002:00010000:2.0:1775764783.630410:0:64085:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xe:0x0]) in obj [0x200003ab4:0xd:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764783.630418:0:64085:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764783.630428:0:64085:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764783.630434:0:64085:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200003ab4:0xd:0x0] suppgids -1 -1: rc 1 00000080:00010000:2.0:1775764783.630439:0:64085:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) 00000080:00010000:2.0:1775764783.630441:0:64085:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) for lock 0x73cfd86cfc17bf52 00000080:00010000:2.0:1775764783.630445:0:64085:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf52 from it ffffa60b01fafcb8 00010000:00010000:2.0:1775764783.630448:0:64085:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764783.630490:0:64085:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764783.630567:0:64085:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764783.630574:0:64085:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764783.630579:0:64085:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:2.0:1775764783.630585:0:64085:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:2.0:1775764783.630589:0:64085:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 1 00000080:00010000:2.0:1775764783.630592:0:64085:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764783.630593:0:64085:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:2.0:1775764783.630596:0:64085:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01fafbf8 00010000:00010000:2.0:1775764783.630597:0:64085:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:2.0:1775764783.630602:0:64085:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:2.0:1775764783.630606:0:64085:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764783.630610:0:64085:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24b.sanity-pcc, flags=25 00000080:00200000:2.0:1775764783.630614:0:64085:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), inode mode 41ed mask 1 00000080:00200000:2.0:1775764783.630616:0:64085:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f24b.sanity-pcc, flags=265 00000080:00200000:2.0:1775764783.630619:0:64085:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xe:0x0](ffff8b2390ec4b08), inode mode 81ff mask 46 00000080:00200008:2.0:1775764783.630626:0:64085:0:(file.c:1057:ll_file_open()) START file f24b.sanity-pcc:[0x200003ab4:0xe:0x0](ffff8b2390ec4b08), flags 154402 00000002:00010000:2.0:1775764783.630637:0:64085:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xe:0x0]) in obj [0x200003ab4:0xd:0x0], intent: open flags 0104240114403 00010000:00010000:2.0:1775764783.630676:0:64085:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf59 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 64085 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.630684:0:64085:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf59 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 64085 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764783.630695:0:64085:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764783.630701:0:64085:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5180 x1862022098643072/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764783.630711:0:64085:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf59 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 64085 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764783.630735:0:64085:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64085:1862022098643072:192.168.202.128@tcp:101:lfs.500 00000100:00100000:2.0:1775764783.630983:0:64085:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c7c00 going to sleep for 11 seconds 00000100:00100000:1.0:1775764783.643380:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764783.643814:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764783.643822:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764783.643829:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8740c00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764783.643833:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764783.643840:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764783.643847:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764783.643875:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bf2f lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052365 expref: -99 pid: 64056 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764783.643908:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764783.643914:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764783.643937:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764783.643943:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17bf52 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e030523dc expref: -99 pid: 64085 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764783.643960:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764783.643961:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764783.643963:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764783.650237:0:64085:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64085:1862022098643072:192.168.202.128@tcp:101:lfs.500 00010000:00010000:2.0:1775764783.650259:0:64085:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CW ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf59 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xd:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030523ea expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.650273:0:64085:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf59 lrc: 4/1,0 mode: --/CW res: [0x200003ab4:0xe:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030523ea expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.650279:0:64085:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf59 lrc: 4/1,0 mode: --/CW res: [0x200003ab4:0xe:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030523ea expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.650288:0:64085:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf59 lrc: 4/1,0 mode: CW/CW res: [0x200003ab4:0xe:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030523ea expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.650296:0:64085:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf59 lrc: 4/1,0 mode: CW/CW res: [0x200003ab4:0xe:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030523ea expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.650306:0:64085:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf59 lrc: 5/1,1 mode: CW/CW res: [0x200003ab4:0xe:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030523ea expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.650310:0:64085:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf59 lrc: 5/1,1 mode: CW/CW res: [0x200003ab4:0xe:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030523ea expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.650315:0:64085:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf59 lrc: 4/0,1 mode: CW/CW res: [0x200003ab4:0xe:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030523ea expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764783.650322:0:64085:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23b9ea5180 x1862022098643072/t34359739223(34359739223) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764799 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.500' uid:500 gid:500 projid:0 00000002:00100000:2.0:1775764783.650331:0:64085:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea5180 x1862022098643072/t34359739223(34359739223) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764799 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.500' uid:500 gid:500 projid:0 00000002:00010000:2.0:1775764783.650342:0:64085:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf59 lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0xe:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030523ea expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.650348:0:64085:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 8 res 8589949620/14 (0 0) 00000080:00200000:2.0:1775764783.650353:0:64085:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab4:0xd:0x0] suppgids -1 -1: rc 0 00000080:00010000:2.0:1775764783.650435:0:64085:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xe:0x0](ffff8b2390ec4b08) for lock 0x73cfd86cfc17bf59 00000080:00010000:2.0:1775764783.650443:0:64085:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf59 from it ffffa60b01fafbf0 00010000:00010000:2.0:1775764783.650446:0:64085:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf59 lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0xe:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030523ea expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.650490:0:64085:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf59 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0xe:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030523ea expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:2.0:1775764783.650510:0:64085:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea5180 x1862022098643072/t34359739223(34359739223) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764799 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.500' uid:500 gid:500 projid:0 00000080:00200080:2.0:1775764783.650586:0:64085:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0xe:0x0](0000000000000000) cmd=c00466fc arg=7fffdc19ad8c 00000080:00200008:2.0:1775764783.650639:0:64085:0:(file.c:447:ll_file_release()) START file f24b.sanity-pcc:[0x200003ab4:0xe:0x0](ffff8b2390ec4b08), flags 154002 00000002:00100000:2.0:1775764783.650672:0:64085:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea5180 x1862022098643072/t34359739223(34359739223) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764799 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.500' uid:500 gid:500 projid:0 00000020:00100000:2.0:1775764783.650682:0:64085:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764783.650687:0:64085:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7100 x1862022098643456/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764783.650708:0:64085:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64085:1862022098643456:192.168.202.128@tcp:35:lfs.500 00000100:00100000:2.0:1775764783.650781:0:64085:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c7c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764783.654595:0:64085:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64085:1862022098643456:192.168.202.128@tcp:35:lfs.500 00000002:00100000:2.0:1775764783.654622:0:64085:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea5180 x1862022098643072/t34359739223(34359739223) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764799 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.500' uid:500 gid:500 projid:0 00000080:00200000:2.0:1775764783.791938:0:64095:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764783.791974:0:64095:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764783.791995:0:64095:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:2.0:1775764783.792013:0:64095:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:2.0:1775764783.792022:0:64095:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764783.792031:0:64095:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764783.792034:0:64095:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:2.0:1775764783.792041:0:64095:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01f5fbf8 00010000:00010000:2.0:1775764783.792045:0:64095:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:2.0:1775764783.792064:0:64095:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:2.0:1775764783.792081:0:64095:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764783.792090:0:64095:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d24b.sanity-pcc, flags=267 00000080:00200000:2.0:1775764783.792097:0:64095:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), inode mode 41ed mask 44 00000080:00200008:2.0:1775764783.792104:0:64095:0:(file.c:1057:ll_file_open()) START file d24b.sanity-pcc:[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), flags 304000 00000002:00010000:2.0:1775764783.792122:0:64095:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xd:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:2.0:1775764783.792180:0:64095:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf60 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64095 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.792186:0:64095:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf60 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64095 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764783.792197:0:64095:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764783.792205:0:64095:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7100 x1862022098643712/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764783.792212:0:64095:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf60 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64095 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764783.792232:0:64095:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64095:1862022098643712:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764783.792396:0:64095:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c7c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764783.807295:0:64095:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64095:1862022098643712:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764783.807321:0:64095:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf60 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64095 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.807334:0:64095:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf60 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64095 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.807340:0:64095:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf60 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64095 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.807345:0:64095:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf60 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64095 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.807352:0:64095:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf60 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64095 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.807356:0:64095:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23ac6ea200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764783.807359:0:64095:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf60 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64095 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.807364:0:64095:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf60 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64095 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.807368:0:64095:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf60 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 64095 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.807373:0:64095:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764783.807378:0:64095:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf60 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64095 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764783.807382:0:64095:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf60 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64095 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764783.807400:0:64095:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23b9ea7100 x1862022098643712/t34359739225(34359739225) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764799 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764783.807413:0:64095:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7100 x1862022098643712/t34359739225(34359739225) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764799 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764783.807426:0:64095:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764783.807444:0:64095:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xd:0x0], updating i_size 512 00000002:00100000:2.0:1775764783.807495:0:64095:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7100 x1862022098643712/t34359739225(34359739225) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764799 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764783.807572:0:64095:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200003ab4:0xd:0x0](ffff8b2390ec3a88) cmd=901066fc arg=7ffcff39e840 00000100:00100000:2.0:1775764783.807622:0:64095:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64095:1862022098643840:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764783.807747:0:64095:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c7c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764783.816174:0:64095:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64095:1862022098643840:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764783.816238:0:64095:0:(file.c:447:ll_file_release()) START file d24b.sanity-pcc:[0x200003ab4:0xd:0x0](ffff8b2390ec3a88), flags 304000 00000002:00100000:2.0:1775764783.816292:0:64095:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea7100 x1862022098643712/t34359739225(34359739225) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764799 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764783.816301:0:64095:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764783.816304:0:64095:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5180 x1862022098644096/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764783.816322:0:64095:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64095:1862022098644096:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764783.816395:0:64095:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23849c7c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764783.819118:0:64095:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64095:1862022098644096:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764783.819150:0:64095:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea7100 x1862022098643712/t34359739225(34359739225) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764799 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1775764783.953718:0:64101:0:(file.c:2454:do_file_read_iter()) START file f24b.sanity-pcc:[0x200003ab4:0xe:0x0], ppos: 0, count: 1024 00000080:00200008:1.0:1775764783.953772:0:64101:0:(file.c:447:ll_file_release()) START file f24b.sanity-pcc:[0x200003ab4:0xe:0x0](ffff8b2390ec4b08), flags 100000 00000002:00100000:1.0:1775764783.953854:0:64101:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864f1880 x1862022098644480/t34359739227(34359739227) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764799 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.500' uid:500 gid:500 projid:0 00000020:00100000:1.0:1775764783.953867:0:64101:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764783.953872:0:64101:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fed80 x1862022098644736/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764783.953898:0:64101:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64101:1862022098644736:192.168.202.128@tcp:35:dd.500 00000100:00100000:1.0:1775764783.954017:0:64101:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b9d6fcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764783.959377:0:64101:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job dd:fcf15866-4739-4595-8fe1-44abde7ee9d7:64101:1862022098644736:192.168.202.128@tcp:35:dd.500 00000002:00100000:1.0:1775764783.959412:0:64101:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864f1880 x1862022098644480/t34359739227(34359739227) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764799 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.500' uid:500 gid:500 projid:0 00010000:00010000:2.0:1775764785.079573:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764786.244822:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2379:1862022098644864:192.168.202.128@tcp:400:kworker.0 00010000:00010000:1.0:1775764787.140622:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764789.178089:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764789.649207:0:64524:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 64524 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764789.649236:0:64524:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64524 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764789.649252:0:64524:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64524 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764789.649283:0:64524:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadc00 pname:cluuid:pid:xid:nid:opc:job lctl:82781e18-e0e3-4f9f-9a2b-041bf0344414:64524:1862022098645504:192.168.202.128@tcp:101:lctl.0 00000100:00100000:1.0:1775764789.649434:0:64524:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654540 going to sleep for 11 seconds 00000100:00100000:1.0:1775764789.662214:0:64524:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadc00 pname:cluuid:pid:xid:nid:opc:job lctl:82781e18-e0e3-4f9f-9a2b-041bf0344414:64524:1862022098645504:192.168.202.128@tcp:101:lctl.0 00010000:00010000:1.0:1775764789.662243:0:64524:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764789.662253:0:64524:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764789.662262:0:64524:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764789.662270:0:64524:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764789.662279:0:64524:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764789.662285:0:64524:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764789.662289:0:64524:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764789.662296:0:64524:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b2382dadc00 x1862022098645504/t0(0) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764805 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lctl.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764789.662306:0:64524:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764789.662321:0:64524:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2387c07800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1775764789.662326:0:64524:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764789.662350:0:64524:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:1.0:1775764789.662355:0:64524:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17bf6e 00000080:00010000:1.0:1775764789.662359:0:64524:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:1.0:1775764789.662361:0:64524:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf6e from it ffffa60b01463cb8 00010000:00010000:1.0:1775764789.662363:0:64524:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764789.662369:0:64524:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764789.662587:0:64524:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764789.662602:0:64524:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764789.662611:0:64524:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:1.0:1775764789.662618:0:64524:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:1.0:1775764789.662623:0:64524:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764789.662626:0:64524:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764789.662628:0:64524:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:1.0:1775764789.662630:0:64524:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01463bb0 00010000:00010000:1.0:1775764789.662632:0:64524:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:1.0:1775764789.662641:0:64524:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:1.0:1775764789.662648:0:64524:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764789.662654:0:64524:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200080:1.0:1775764789.662670:0:64524:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=8080667f arg=7fff2d3ac830 00000080:00200008:1.0:1775764789.662680:0:64524:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200000:1.0:1775764789.662970:0:64524:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764789.662976:0:64524:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764789.662982:0:64524:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1775764789.662988:0:64524:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:1.0:1775764789.662992:0:64524:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764789.662994:0:64524:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764789.663008:0:64524:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:1.0:1775764789.663012:0:64524:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01463bb0 00010000:00010000:1.0:1775764789.663015:0:64524:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1775764789.663023:0:64524:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:1.0:1775764789.663032:0:64524:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764789.663037:0:64524:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200080:1.0:1775764789.663045:0:64524:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=8080667f arg=7fff2d3ab710 00000080:00200008:1.0:1775764789.663052:0:64524:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200000:1.0:1775764789.663176:0:64524:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764789.663184:0:64524:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764789.663191:0:64524:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:1.0:1775764789.663199:0:64524:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:1.0:1775764789.663205:0:64524:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764789.663209:0:64524:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764789.663211:0:64524:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:1.0:1775764789.663214:0:64524:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01463bb0 00010000:00010000:1.0:1775764789.663217:0:64524:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:1.0:1775764789.663224:0:64524:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00000080:00200000:1.0:1775764789.663232:0:64524:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764789.663236:0:64524:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 100000 00000080:00200008:1.0:1775764789.663443:0:64524:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 100000 00000080:00200008:2.0:1775764789.790894:0:64004:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 100000 00000080:00200008:2.0:1775764790.051816:0:64542:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764790.051832:0:64542:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:2.0:1775764790.051857:0:64542:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764790.051878:0:64542:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764790.051894:0:64542:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764790.051903:0:64542:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764790.051912:0:64542:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:2.0:1775764790.051915:0:64542:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17bf6e 00000080:00010000:2.0:1775764790.051952:0:64542:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf6e from it ffffa60b0146bcb8 00010000:00010000:2.0:1775764790.051957:0:64542:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764790.051966:0:64542:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200008:2.0:1775764790.052857:0:64542:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764790.052876:0:64542:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764790.052893:0:64542:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764790.052907:0:64542:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00010000:00010000:2.0:1775764790.052919:0:64542:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00000080:00200000:2.0:1775764790.052927:0:64542:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764790.052933:0:64542:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764790.052936:0:64542:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:2.0:1775764790.052942:0:64542:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b0146bcb8 00010000:00010000:2.0:1775764790.052945:0:64542:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00010000:00010000:2.0:1775764790.052951:0:64542:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:1.0:1775764790.200215:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764791.360722:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2379:1862022098646144:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764791.360742:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2379:1862022098646016:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764791.360754:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098646272:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764791.360765:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098646528:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764791.380394:0:2379:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b2389398000: committing for last_committed 34359738455 gen 1 00000100:00100000:2.0:1775764791.380429:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098646400:192.168.202.128@tcp:400:kworker.0 00010000:00010000:1.0:1775764792.254751:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1775764792.375801:0:10503:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0xc:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764792.375819:0:10503:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0xc:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764792.375823:0:10503:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1775764792.375830:0:10503:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764792.375891:0:10503:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0xc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1775764794.300636:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1775764796.206492:0:64801:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1775764796.206508:0:64801:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764796.206530:0:64801:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764796.206550:0:64801:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:2.0:1775764796.206567:0:64801:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00000080:00200000:2.0:1775764796.206576:0:64801:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764796.206586:0:64801:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764796.206589:0:64801:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:2.0:1775764796.206596:0:64801:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01d8bcb8 00010000:00010000:2.0:1775764796.206599:0:64801:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:2.0:1775764796.206608:0:64801:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00000080:00200008:2.0:1775764796.207150:0:64801:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1775764796.207156:0:64801:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:2.0:1775764796.207167:0:64801:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764796.207179:0:64801:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764796.207194:0:64801:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764796.207201:0:64801:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764796.207206:0:64801:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:2.0:1775764796.207209:0:64801:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17bf6e 00000080:00010000:2.0:1775764796.207213:0:64801:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf6e from it ffffa60b01d8bcb8 00010000:00010000:2.0:1775764796.207216:0:64801:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764796.207224:0:64801:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764796.207800:0:64801:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764796.207812:0:64801:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764796.207885:0:64801:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00010000:00010000:2.0:1775764796.207898:0:64801:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00000080:00200000:2.0:1775764796.207906:0:64801:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764796.207912:0:64801:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764796.207914:0:64801:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:2.0:1775764796.207918:0:64801:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01d8bbb0 00010000:00010000:2.0:1775764796.207921:0:64801:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00010000:00010000:2.0:1775764796.207930:0:64801:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00000080:00200000:2.0:1775764796.207951:0:64801:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764796.207961:0:64801:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 100000 00010000:00010000:2.0:1775764796.245039:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764796.245049:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17bf75 lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03052461 expref: -99 pid: 64802 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.245059:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5fe00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764796.245065:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17bf75 lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03052461 expref: -99 pid: 64802 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.245073:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17bf75 lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03052461 expref: -99 pid: 64802 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.245080:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17bf75 lrc: 3/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052461 expref: -99 pid: 64802 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.245118:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e000/0x73cfd86cfc17baf9 lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305144d expref: -99 pid: 61408 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764796.245156:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17bf75 lrc: 2/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052461 expref: -99 pid: 64802 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.245163:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764796.245182:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17bf75 lrc: 1/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052461 expref: -99 pid: 64802 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.245189:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17bf75 lrc: 0/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052461 expref: -99 pid: 64802 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.245201:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764796.245202:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764796.245204:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764796.245211:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764796.245228:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098647040:192.168.202.128@tcp:103:ldlm_bl.0 00000080:00200000:2.0:1775764796.402829:0:64810:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764796.402860:0:64810:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764796.402882:0:64810:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:2.0:1775764796.402897:0:64810:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00000080:00200000:2.0:1775764796.402905:0:64810:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764796.402914:0:64810:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764796.402918:0:64810:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:2.0:1775764796.402926:0:64810:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01db3bb0 00010000:00010000:2.0:1775764796.402931:0:64810:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:2.0:1775764796.402951:0:64810:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:2.0:1775764796.402973:0:64810:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764796.402983:0:64810:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200080:2.0:1775764796.403018:0:64810:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=8080667f arg=7ffd7205d340 00000080:00200008:2.0:1775764796.403037:0:64810:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200000:1.0:1775764796.433546:0:64811:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764796.433576:0:64811:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764796.433594:0:64811:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:1.0:1775764796.433610:0:64811:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:1.0:1775764796.433619:0:64811:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764796.433627:0:64811:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764796.433630:0:64811:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:1.0:1775764796.433637:0:64811:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01dc3e18 00010000:00010000:1.0:1775764796.433642:0:64811:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:1.0:1775764796.433650:0:64811:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:1.0:1775764796.433665:0:64811:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 101 00000080:00200000:1.0:1775764796.433676:0:64811:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764796.433682:0:64811:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764796.433689:0:64811:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1775764796.433697:0:64811:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:1.0:1775764796.433703:0:64811:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764796.433707:0:64811:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764796.433710:0:64811:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:1.0:1775764796.433713:0:64811:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01dc3be0 00010000:00010000:1.0:1775764796.433716:0:64811:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1775764796.433722:0:64811:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000080:00200000:1.0:1775764796.433730:0:64811:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764796.433743:0:64811:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=d25.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), flags=1536 00000080:00200000:1.0:1775764796.433746:0:64811:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764796.433751:0:64811:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764796.433757:0:64811:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:1.0:1775764796.433765:0:64811:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000080:00200000:1.0:1775764796.433770:0:64811:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764796.433774:0:64811:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764796.433776:0:64811:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:1.0:1775764796.433779:0:64811:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01dc3ce0 00010000:00010000:1.0:1775764796.433782:0:64811:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:1.0:1775764796.433795:0:64811:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00000080:00200000:1.0:1775764796.433803:0:64811:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:1.0:1775764796.433808:0:64811:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764796.433813:0:64811:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764796.433819:0:64811:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00010000:00010000:1.0:1775764796.433826:0:64811:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00000080:00200000:1.0:1775764796.433833:0:64811:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764796.433836:0:64811:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764796.433839:0:64811:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bec6 00000080:00010000:1.0:1775764796.433842:0:64811:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bec6 from it ffffa60b01dc3dc8 00010000:00010000:1.0:1775764796.433845:0:64811:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00010000:00010000:1.0:1775764796.433851:0:64811:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00000080:00200000:1.0:1775764796.433858:0:64811:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:1.0:1775764796.433862:0:64811:0:(namei.c:2226:do_mkdir()) VFS Op:name=d25.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08) 00010000:00010000:1.0:1775764796.433893:0:64811:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00010000:00010000:1.0:1775764796.433963:0:64811:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00010000:00010000:1.0:1775764796.433971:0:64811:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764796.433974:0:64811:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eae00/0x73cfd86cfc17bec6 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052200 expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00000020:00100000:1.0:1775764796.434000:0:64811:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1775764796.434014:0:64811:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fed80 x1862022098647296/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 560/984 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764796.434041:0:64811:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job mkdir:fcf15866-4739-4595-8fe1-44abde7ee9d7:64811:1862022098647296:192.168.202.128@tcp:36:mkdir.0 00000100:00100000:1.0:1775764796.434148:0:64811:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654780 going to sleep for 11 seconds 00000100:00100000:1.0:1775764796.439020:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119063936 00000100:00100000:1.0:1775764796.439031:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119063936 00000100:00100000:1.0:1775764796.439091:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 257 00000100:00100000:1.0:1775764796.439102:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daf480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42324:x1862022119063936:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764796.439123:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764796.439204:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764796.439211:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daf480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42324:x1862022119063936:12345-192.168.202.128@tcp:104: Request processed in 111us (301us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764796.439219:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 257 00010000:00010000:2.0:1775764796.440000:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764796.440009:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.440020:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8740800 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764796.440025:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.440035:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.440044:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.440108:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 2/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.440116:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764796.440282:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.440291:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8740800/0x73cfd86cfc17bf6e lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x13 rrc: 2 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03052445 expref: -99 pid: 64524 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.440320:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764796.440323:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764796.440324:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764796.459109:0:64811:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job mkdir:fcf15866-4739-4595-8fe1-44abde7ee9d7:64811:1862022098647296:192.168.202.128@tcp:36:mkdir.0 00000080:00200000:1.0:1775764796.459167:0:64811:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988) 00000080:00200000:1.0:1775764796.459177:0:64811:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xf:0x0], updating i_size 512 00000080:00200000:2.0:1775764796.463320:0:64571:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764796.463348:0:64571:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764796.463420:0:64571:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 64571 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.463432:0:64571:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64571 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.463443:0:64571:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64571 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764796.463515:0:64571:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098647552:192.168.202.128@tcp:101:bash.0 00000100:00100000:2.0:1775764796.463610:0:64571:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394fb40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764796.469541:0:64571:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098647552:192.168.202.128@tcp:101:bash.0 00010000:00010000:2.0:1775764796.469572:0:64571:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.469588:0:64571:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.469600:0:64571:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.469612:0:64571:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.469625:0:64571:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.469633:0:64571:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.469641:0:64571:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764796.469653:0:64571:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea7100 x1862022098647552/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764812 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764796.469667:0:64571:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.469681:0:64571:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764796.469688:0:64571:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764796.469720:0:64571:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764796.469726:0:64571:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:2.0:1775764796.469732:0:64571:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764796.469735:0:64571:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01c0bbf8 00010000:00010000:2.0:1775764796.469738:0:64571:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.469747:0:64571:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764796.469776:0:64571:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764796.469797:0:64571:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=d25.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), flags=25 00000080:00200000:2.0:1775764796.469802:0:64571:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=d25.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=0 00000002:00010000:2.0:1775764796.469830:0:64571:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d25.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:2.0:1775764796.469874:0:64571:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf83 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64571 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.469882:0:64571:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf83 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64571 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.469891:0:64571:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf83 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64571 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764796.469913:0:64571:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098647680:192.168.202.128@tcp:101:bash.0 00000100:00100000:2.0:1775764796.470011:0:64571:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394fb40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764796.477364:0:64571:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098647680:192.168.202.128@tcp:101:bash.0 00010000:00010000:2.0:1775764796.477391:0:64571:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf83 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524a0 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.477414:0:64571:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf83 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524a0 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.477422:0:64571:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf83 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524a0 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.477434:0:64571:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf83 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030524a0 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.477448:0:64571:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf83 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524a0 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.477485:0:64571:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf83 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030524a0 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.477493:0:64571:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf83 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030524a0 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.477500:0:64571:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf83 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030524a0 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764796.477512:0:64571:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=b, status=0 req@ffff8b23b9ea7100 x1862022098647680/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/584 e 0 to 0 dl 1775764812 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764796.477524:0:64571:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf83 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524a0 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.477535:0:64571:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/15 (0 0) 00000080:00200000:2.0:1775764796.477541:0:64571:0:(namei.c:953:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764796.477561:0:64571:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xf:0x0], updating i_size 512 00000080:00200000:2.0:1775764796.477564:0:64571:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0xf:0x0](ffff8b2389a61988): rc = 0 00000080:00010000:2.0:1775764796.477570:0:64571:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xf:0x0](ffff8b2389a61988) for lock 0x73cfd86cfc17bf83 00000080:00010000:2.0:1775764796.477582:0:64571:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xf:0x0](ffff8b2389a61988) 00000080:00010000:2.0:1775764796.477586:0:64571:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf83 from it ffffa60b01c0bac8 00010000:00010000:2.0:1775764796.477588:0:64571:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf83 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030524a0 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.477603:0:64571:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf83 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524a0 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764796.477634:0:64571:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764796.477652:0:64571:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988), inode mode 41ed mask 3 00000080:00200000:2.0:1775764796.477657:0:64571:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f25.sanity-pcc, dir=[0x200003ab4:0xf:0x0](ffff8b2389a61988), file ffff8b2383ef0300, open_flags 8241, mode 81b6 opened 0 00000080:00200000:2.0:1775764796.477665:0:64571:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f25.sanity-pcc, dir=[0x200003ab4:0xf:0x0](ffff8b2389a61988), intent=open|creat 00000002:00010000:2.0:1775764796.477688:0:64571:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f25.sanity-pcc,[0x200003ab4:0x10:0x0]) in obj [0x200003ab4:0xf:0x0], intent: open|creat flags 0101102 00010000:00010000:2.0:1775764796.477737:0:64571:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 3/0,1 mode: --/CW res: [0x200003ab4:0xf:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64571 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.477744:0:64571:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 3/0,1 mode: --/CW res: [0x200003ab4:0xf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64571 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764796.477755:0:64571:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764796.477761:0:64571:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7100 x1862022098647936/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764796.477771:0:64571:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 3/0,1 mode: --/CW res: [0x200003ab4:0xf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64571 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764796.477794:0:64571:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098647936:192.168.202.128@tcp:101:bash.0 00000100:00100000:2.0:1775764796.477949:0:64571:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394fb40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764796.484073:0:64571:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098647936:192.168.202.128@tcp:101:bash.0 00010000:00010000:2.0:1775764796.484101:0:64571:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 4/0,1 mode: --/CW res: [0x200003ab4:0xf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.484127:0:64571:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.484135:0:64571:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.484147:0:64571:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.484158:0:64571:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.484172:0:64571:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.484180:0:64571:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.484186:0:64571:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764796.484198:0:64571:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8b23b9ea7100 x1862022098647936/t34359739232(34359739232) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764812 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764796.484211:0:64571:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7100 x1862022098647936/t34359739232(34359739232) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764812 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764796.484224:0:64571:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764796.484237:0:64571:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.484246:0:64571:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/16 (0 0) 00000080:00200000:2.0:1775764796.484252:0:64571:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200003ab4:0xf:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764796.484267:0:64571:0:(namei.c:1620:ll_create_it()) VFS Op:name=f25.sanity-pcc, dir=[0x200003ab4:0xf:0x0](ffff8b2389a61988), intent=open|creat 00000080:00200000:2.0:1775764796.484294:0:64571:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab4:0x10:0x0](ffff8b2390ec1148) 00000080:00200000:2.0:1775764796.484414:0:64571:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x10:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1775764796.484425:0:64571:0:(namei.c:1586:ll_create_node()) setting l_ast_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) 00000080:00010000:2.0:1775764796.484429:0:64571:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) for lock 0x73cfd86cfc17bf8a 00000080:00010000:2.0:1775764796.484438:0:64571:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) 00000080:00200008:2.0:1775764796.484473:0:64571:0:(file.c:1057:ll_file_open()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148), flags 101101 00000002:00100000:2.0:1775764796.484491:0:64571:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7100 x1862022098647936/t34359739232(34359739232) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764812 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764796.484524:0:64571:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf8a from it ffff8b238504ce40 00010000:00010000:2.0:1775764796.484529:0:64571:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.484538:0:64571:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764796.484576:0:64571:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x10:0x0](ffff8b2390ec1148), inode mode 81a4 mask 40 00000080:00200008:2.0:1775764796.484664:0:64571:0:(file.c:2612:do_file_write_iter()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0], ppos: 0, count: 21 00000080:00200000:2.0:1775764796.484676:0:64571:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 21 bytes 00000080:00200000:2.0:1775764796.484683:0:64571:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 21 00000080:00200000:2.0:1775764796.484695:0:64571:0:(file.c:2058:ll_file_io_generic()) f25.sanity-pcc: write ppos: 0, bytes: 21 00000080:00200000:2.0:1775764796.484700:0:64571:0:(file.c:1815:ll_io_set_mirror()) f25.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764796.484707:0:64571:0:(cl_io.c:214:cl_io_rw_init()) header@00000000ab8aad76[0x110, 1, [0x200003ab4:0x10:0x0] hash] 00000020:00200000:2.0:1775764796.484709:0:64571:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 21) 0 0 00000080:00200000:2.0:1775764796.484714:0:64571:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764796.484726:0:64571:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:2.0:1775764796.484733:0:64571:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 21) 21, 21 00020000:00200000:2.0:1775764796.484737:0:64571:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764796.484765:0:64571:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1775764796.484778:0:64571:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764796.484782:0:64571:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8030, osc lock ffff8b238a3f8850, flags 0x0 00000100:00100000:1.0:1775764796.485129:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2376:1862022098648192:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764796.485143:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098648448:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764796.485153:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2376:1862022098648064:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764796.485164:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098648320:192.168.202.128@tcp:400:kworker.0 00010000:00010000:2.0:1775764796.485865:0:64571:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.485881:0:64571:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.485893:0:64571:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764796.485919:0:64571:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098648576:192.168.202.128@tcp:101:bash.0 00000100:00100000:2.0:1775764796.486010:0:64571:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764796.497886:0:64571:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098648576:192.168.202.128@tcp:101:bash.0 00010000:00010000:2.0:1775764796.497916:0:64571:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.497950:0:64571:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.497965:0:64571:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.497980:0:64571:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1775764796.497993:0:64571:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.498004:0:64571:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.498012:0:64571:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764796.498039:0:64571:0:(vvp_io.c:1286:vvp_io_write_start()) f25.sanity-pcc: write [0, 21) append 0 append lockpos 0 00000080:00200000:2.0:1775764796.498055:0:64571:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 21 bytes 00000080:00200000:2.0:1775764796.498106:0:64571:0:(rw26.c:858:ll_write_end()) pos 0, len 21, copied 21 00000080:00200000:2.0:1775764796.498117:0:64571:0:(rw26.c:916:ll_write_end()) page@000000008035b36e[3 000000007c1d912c 2 1 0000000000567e49] 00000080:00200000:2.0:1775764796.498121:0:64571:0:(rw26.c:916:ll_write_end()) vmpage @0000000046e80d20 17ffffc0001001 3:0 ffff8b23baeaea80 0 lru 00000080:00200000:2.0:1775764796.498128:0:64571:0:(rw26.c:916:ll_write_end()) osc-page@00000000b24ed86e 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 000000008819327c 000000003d8bbb9f > 3< 0 0 > 4< 0 0 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764796.498130:0:64571:0:(rw26.c:916:ll_write_end()) end page@000000008035b36e 00000080:00200000:2.0:1775764796.498132:0:64571:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764796.498137:0:64571:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 21 prio 0 00000008:00010000:2.0:1775764796.498154:0:64571:0:(osc_cache.c:848:osc_extent_find()) ### extent: 00000000fa56d242 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764796.498172:0:64571:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2390ec12f8, count 1, dirtied 1 00000080:00200000:2.0:1775764796.498176:0:64571:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 21 bytes, tot: 21 00000080:00200000:2.0:1775764796.498179:0:64571:0:(vvp_io.c:1353:vvp_io_write_start()) f25.sanity-pcc: write bytes 0, result: 21 00000020:00010000:2.0:1775764796.498195:0:64571:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764796.498199:0:64571:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764796.498202:0:64571:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8030 (ffff8b23812ec5a8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764796.498207:0:64571:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.498216:0:64571:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1775764796.498232:0:64571:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764796.498236:0:64571:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8030 (ffff8b23812eca28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764796.498243:0:64571:0:(vvp_io.c:517:vvp_io_advance()) advancing 21 bytes 00000080:00200000:2.0:1775764796.498249:0:64571:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:2.0:1775764796.498294:0:64571:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764796.498299:0:64571:0:(file.c:2206:ll_file_io_generic()) f25.sanity-pcc: 2 io complete with rc: 0, result: 21, restart: 0 00000080:00200000:2.0:1775764796.498307:0:64571:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x10:0x0] updating i_size 21 i_blocks 1 00000080:00200000:2.0:1775764796.498319:0:64571:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 21 00000080:00200008:2.0:1775764796.498366:0:64571:0:(file.c:447:ll_file_release()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148), flags 100001 00000002:00100000:2.0:1775764796.498429:0:64571:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea7100 x1862022098647936/t34359739232(34359739232) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764812 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764796.498440:0:64571:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764796.498445:0:64571:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7800 x1862022098648832/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764796.498508:0:64571:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098648832:192.168.202.128@tcp:35:bash.0 00000100:00100000:2.0:1775764796.498571:0:64571:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764796.501112:0:64571:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098648832:192.168.202.128@tcp:35:bash.0 00000002:00100000:2.0:1775764796.501146:0:64571:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea7100 x1862022098647936/t34359739232(34359739232) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764812 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1775764796.571140:0:64817:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764796.571156:0:64817:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764796.571181:0:64817:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764796.571202:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764796.571218:0:64817:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764796.571226:0:64817:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764796.571234:0:64817:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764796.571238:0:64817:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:2.0:1775764796.571245:0:64817:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01dd3cb8 00010000:00010000:2.0:1775764796.571249:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764796.571259:0:64817:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764796.571288:0:64817:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764796.571293:0:64817:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764796.571298:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764796.571305:0:64817:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764796.571311:0:64817:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764796.571315:0:64817:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764796.571317:0:64817:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:2.0:1775764796.571320:0:64817:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01dd3b58 00010000:00010000:2.0:1775764796.571322:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764796.571329:0:64817:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764796.571337:0:64817:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764796.571343:0:64817:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d25.sanity-pcc, flags=8 00000080:00200008:2.0:1775764796.571351:0:64817:0:(file.c:6226:ll_getattr_dentry()) START file d25.sanity-pcc:[0x200003ab4:0xf:0x0](ffff8b2389a61988), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764796.571355:0:64817:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988),name=d25.sanity-pcc 00000002:00010000:2.0:1775764796.571365:0:64817:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xf:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764796.571371:0:64817:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 30 res 8589949620/15 (0 0) 00010000:00010000:2.0:1775764796.571408:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf98 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64817 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.571416:0:64817:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf98 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64817 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.571425:0:64817:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf98 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64817 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764796.571449:0:64817:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64817:1862022098648960:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764796.571579:0:64817:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764796.579406:0:64817:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64817:1862022098648960:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764796.579433:0:64817:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf98 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524d8 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.579450:0:64817:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf98 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524d8 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.579637:0:64817:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf98 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524d8 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.579654:0:64817:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf98 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030524d8 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.579666:0:64817:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf98 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524d8 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.579681:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf98 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030524d8 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.579689:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf98 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030524d8 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.579694:0:64817:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf98 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030524d8 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764796.579702:0:64817:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea4380 x1862022098648960/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/584 e 0 to 0 dl 1775764812 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764796.579712:0:64817:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf98 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524d8 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.579721:0:64817:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/15 (0 0) 00000080:00200000:2.0:1775764796.579727:0:64817:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764796.579747:0:64817:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xf:0x0], updating i_size 512 00000080:00010000:2.0:1775764796.579752:0:64817:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xf:0x0](ffff8b2389a61988) for lock 0x73cfd86cfc17bf98 00000080:00010000:2.0:1775764796.579759:0:64817:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xf:0x0](ffff8b2389a61988) 00000080:00010000:2.0:1775764796.579761:0:64817:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf98 from it ffffa60b01dd3cb8 00010000:00010000:2.0:1775764796.579763:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf98 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030524d8 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.579769:0:64817:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf98 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524d8 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764796.579816:0:64817:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764796.579831:0:64817:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764796.579840:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.579850:0:64817:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764796.579856:0:64817:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764796.579860:0:64817:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764796.579863:0:64817:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:2.0:1775764796.579866:0:64817:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01dd3b58 00010000:00010000:2.0:1775764796.579869:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.579882:0:64817:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764796.579891:0:64817:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764796.579898:0:64817:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d25.sanity-pcc, flags=24 00000080:00200000:2.0:1775764796.579902:0:64817:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764796.579905:0:64817:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f25.sanity-pcc, flags=8 00000080:00200008:2.0:1775764796.579912:0:64817:0:(file.c:6226:ll_getattr_dentry()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764796.579916:0:64817:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x10:0x0](ffff8b2390ec1148),name=f25.sanity-pcc 00000002:00010000:2.0:1775764796.579928:0:64817:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x10:0x0]) in obj [0x200003ab4:0xf:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764796.579946:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764796.579955:0:64817:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764796.579962:0:64817:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200003ab4:0xf:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764796.579966:0:64817:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) 00000080:00010000:2.0:1775764796.579968:0:64817:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) for lock 0x73cfd86cfc17bf8a 00000080:00010000:2.0:1775764796.579972:0:64817:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf8a from it ffffa60b01dd3cb8 00010000:00010000:2.0:1775764796.579975:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764796.579982:0:64817:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764796.580046:0:64817:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1775764796.580055:0:64817:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x10:0x0] 00000008:00010000:2.0:1775764796.580078:0:64817:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764796.580081:0:64817:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8030, osc lock ffff8b238a3f8850, flags 0x400001002 00010000:00010000:2.0:1775764796.580094:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764796.580102:0:64817:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1775764796.580110:0:64817:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764796.580113:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764796.580118:0:64817:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1775764796.580124:0:64817:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764796.580132:0:64817:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x10:0x0] updating i_size 21 i_blocks 1 00000020:00010000:2.0:1775764796.580135:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764796.580137:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764796.580139:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8030 (ffff8b23812ec5a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764796.580141:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764796.580143:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8030 (ffff8b23812eca28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764796.580154:0:64817:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764796.580179:0:64817:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764796.580184:0:64817:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764796.580188:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764796.580193:0:64817:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764796.580197:0:64817:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764796.580199:0:64817:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764796.580201:0:64817:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:2.0:1775764796.580204:0:64817:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01dd3bf8 00010000:00010000:2.0:1775764796.580205:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764796.580210:0:64817:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764796.580214:0:64817:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764796.580216:0:64817:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d25.sanity-pcc, flags=25 00000080:00200000:2.0:1775764796.580218:0:64817:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764796.580220:0:64817:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f25.sanity-pcc, flags=265 00000080:00200000:2.0:1775764796.580222:0:64817:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x10:0x0](ffff8b2390ec1148), inode mode 81a4 mask 44 00000080:00200008:2.0:1775764796.580228:0:64817:0:(file.c:1057:ll_file_open()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148), flags 100000 00000002:00010000:2.0:1775764796.580237:0:64817:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x10:0x0]) in obj [0x200003ab4:0xf:0x0], intent: open flags 04240100001 00010000:00010000:2.0:1775764796.580273:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17bf9f lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 64817 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.580279:0:64817:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17bf9f lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 64817 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764796.580294:0:64817:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764796.580301:0:64817:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4380 x1862022098649216/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764796.580309:0:64817:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17bf9f lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 64817 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764796.580330:0:64817:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64817:1862022098649216:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764796.580730:0:64817:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764796.583247:0:64817:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64817:1862022098649216:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764796.583279:0:64817:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17bf9f lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524e6 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.583290:0:64817:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17bf9f lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524e6 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.583301:0:64817:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17bf9f lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030524e6 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.583310:0:64817:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17bf9f lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524e6 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764796.583320:0:64817:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23b9ea4380 x1862022098649216/t34359739234(34359739234) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764812 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764796.583329:0:64817:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4380 x1862022098649216/t34359739234(34359739234) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764812 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764796.583341:0:64817:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17bf9f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524e6 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764796.583352:0:64817:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17bf9f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524e6 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.583358:0:64817:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/16 (0 0) 00000080:00200000:2.0:1775764796.583364:0:64817:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab4:0xf:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764796.583667:0:64817:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x10:0x0]: layout version change: 0 -> 0 00000080:00010000:2.0:1775764796.583676:0:64817:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) for lock 0x73cfd86cfc17bf9f 00000080:00010000:2.0:1775764796.583682:0:64817:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf9f from it ffffa60b01dd3bf0 00010000:00010000:2.0:1775764796.583684:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17bf9f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030524e6 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.583692:0:64817:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17bf9f lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030524e6 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00100000:2.0:1775764796.583713:0:64817:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4380 x1862022098649216/t34359739234(34359739234) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764812 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764796.583798:0:64817:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x10:0x0](ffff8b2390ec1148) cmd=400866fc arg=7ffd6ce843e8 00000080:00010000:2.0:1775764796.583831:0:64817:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x10:0x0](ffff8b2390ec1148) 00000002:00010000:2.0:1775764796.583840:0:64817:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x10:0x0]) in obj [0x200003ab4:0x10:0x0], intent: layout flags 02 00010000:00010000:2.0:1775764796.583856:0:64817:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17bf9f lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e030524e6 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764796.583882:0:64817:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x10:0x0]: losing layout lock 00010000:00010000:2.0:1775764796.583904:0:64817:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764796.583920:0:64817:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x10:0x0]: losing layout lock 00010000:00010000:2.0:1775764796.583962:0:64817:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17bf9f lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030524e6 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764796.583969:0:64817:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764796.583973:0:64817:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1775764796.583975:0:64817:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17bf9f lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030524e6 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764796.583984:0:64817:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb800/0x73cfd86cfc17bf8a lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030524bc expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764796.584004:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x10:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 64817 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.584009:0:64817:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x10:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64817 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764796.584016:0:64817:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764796.584020:0:64817:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4000 x1862022098649472/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 384/880 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764796.584026:0:64817:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x10:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64817 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764796.584041:0:64817:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64817:1862022098649472:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764796.584146:0:64817:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764796.586625:0:64817:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64817:1862022098649472:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764796.586650:0:64817:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 4/0,1 mode: --/EX res: [0x200003ab4:0x10:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.586658:0:64817:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 4/0,1 mode: --/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.586667:0:64817:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.586674:0:64817:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.586684:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.586689:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.586694:0:64817:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764796.586701:0:64817:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea4000 x1862022098649472/t34359739235(34359739235) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 384/600 e 0 to 0 dl 1775764812 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764796.586714:0:64817:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764796.586726:0:64817:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.586735:0:64817:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/16 (0 0) 00000080:00200000:2.0:1775764796.586739:0:64817:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x10:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764796.586745:0:64817:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) for lock 0x73cfd86cfc17bfa6 00000080:00010000:2.0:1775764796.586751:0:64817:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bfa6 from it ffffa60b01dd3c10 00010000:00010000:2.0:1775764796.586753:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.586759:0:64817:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764796.586774:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764796.586782:0:64817:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764796.586787:0:64817:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x10:0x0](000000007b174fdb) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764796.586877:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764796.586886:0:64817:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764796.586914:0:64817:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764796.586924:0:64817:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:2.0:1775764796.586976:0:64817:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 00000000fa56d242 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:2.0:1775764796.586990:0:64817:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000fa56d242 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764796.587034:0:64817:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:2.0:1775764796.587101:0:64817:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea7b80 - object 0x280000400:283 offset 0<>21 00000080:00200000:2.0:1775764796.590102:0:64817:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764796.590195:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764796.590207:0:64817:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764796.590215:0:64817:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x10:0x0](000000007b174fdb) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764796.590272:0:64817:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764796.590352:0:64817:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764796.590531:0:64817:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:2.0:1775764796.590551:0:64817:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 21 blocks 1 [cma]time 1775764796 1775764796 0 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764796.590618:0:64817:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x10:0x0]: layout version change: 4294967294 -> 2 00010000:00010000:2.0:1775764796.590630:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764796.590640:0:64817:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:2.0:1775764796.591198:0:64817:0:(file.c:2454:do_file_read_iter()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0], ppos: 0, count: 1048576 00000080:00200000:2.0:1775764796.591222:0:64817:0:(rw.c:1928:ll_readpage()) fast read pgno: 0 00000080:00200000:2.0:1775764796.591242:0:64817:0:(file.c:2058:ll_file_io_generic()) f25.sanity-pcc: read ppos: 0, bytes: 1048576 00000080:00200000:2.0:1775764796.591246:0:64817:0:(file.c:1815:ll_io_set_mirror()) f25.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764796.591250:0:64817:0:(cl_io.c:214:cl_io_rw_init()) header@00000000ab8aad76[0x110, 1, [0x200003ab4:0x10:0x0] hash] 00000020:00200000:2.0:1775764796.591252:0:64817:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 1048576) 0 0 00000080:00200000:2.0:1775764796.591255:0:64817:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764796.591266:0:64817:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x10:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764796.591268:0:64817:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764796.591355:0:64817:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:2.0:1775764796.591358:0:64817:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764796.591436:0:64817:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:2.0:1775764796.591447:0:64817:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764796.591493:0:64817:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8b30, osc lock ffff8b238a3f8850, flags 0x400000000000000 00010000:00010000:2.0:1775764796.591511:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.591523:0:64817:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.591533:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:2.0:1775764796.591543:0:64817:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 21, setting rss=21; kms=21, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 5/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.591553:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.591560:0:64817:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764796.591569:0:64817:0:(vvp_io.c:826:vvp_io_read_start()) f25.sanity-pcc: read [0, 1048576) 00000001:00010000:2.0:1775764796.591585:0:64817:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x10:0x0] 00000008:00010000:2.0:1775764796.591590:0:64817:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764796.591592:0:64817:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c84b0, osc lock ffff8b238a3f8000, flags 0x400000400001002 00010000:00010000:2.0:1775764796.591599:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.591609:0:64817:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:2.0:1775764796.591618:0:64817:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764796.591621:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.591628:0:64817:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:2.0:1775764796.591639:0:64817:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764796.591646:0:64817:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x10:0x0] updating i_size 21 i_blocks 1 00000020:00010000:2.0:1775764796.591649:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764796.591652:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764796.591655:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c84b0 (ffff8b23812ec5a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764796.591659:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764796.591662:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c84b0 (ffff8b23812eca28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764796.591669:0:64817:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:0 00000008:00010000:2.0:1775764796.591713:0:64817:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b2383c9e780 00000008:00010000:2.0:1775764796.591754:0:46654:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b2383c9e780 00010000:00010000:2.0:1775764796.591770:0:46654:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:2.0:1775764796.591805:0:46654:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea4e00 - object 0x280000400:283 offset 0<>4096 00000020:00010000:2.0:1775764796.594879:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764796.594883:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764796.594887:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8b30 (ffff8b23812ec5a8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764796.594895:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.594910:0:64817:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:2.0:1775764796.594924:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764796.594926:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8b30 (ffff8b23812eca28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764796.594948:0:64817:0:(vvp_io.c:517:vvp_io_advance()) advancing 21 bytes 00000080:00200000:2.0:1775764796.594980:0:64817:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764796.594985:0:64817:0:(file.c:2206:ll_file_io_generic()) f25.sanity-pcc: 1 io complete with rc: 0, result: 21, restart: 0 00000080:00200000:2.0:1775764796.594992:0:64817:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 21 00000080:00200008:2.0:1775764796.595103:0:64817:0:(file.c:2454:do_file_read_iter()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0], ppos: 21, count: 1048576 00000080:00200000:2.0:1775764796.595120:0:64817:0:(file.c:2058:ll_file_io_generic()) f25.sanity-pcc: read ppos: 21, bytes: 1048576 00000080:00200000:2.0:1775764796.595123:0:64817:0:(file.c:1815:ll_io_set_mirror()) f25.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764796.595128:0:64817:0:(cl_io.c:214:cl_io_rw_init()) header@00000000ab8aad76[0x110, 2, [0x200003ab4:0x10:0x0] hash] 00000020:00200000:2.0:1775764796.595129:0:64817:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [21, 1048597) 0 0 00000080:00200000:2.0:1775764796.595132:0:64817:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764796.595146:0:64817:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x10:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764796.595148:0:64817:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764796.595154:0:64817:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 21 chunk: [21, 1048597) 1048597, 1048576 00020000:00200000:2.0:1775764796.595157:0:64817:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764796.595177:0:64817:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 256] 00000008:00010000:2.0:1775764796.595186:0:64817:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764796.595189:0:64817:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8b30, osc lock ffff8b238a3f8850, flags 0x400000000000000 00010000:00010000:2.0:1775764796.595207:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764796.595217:0:64817:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1052671) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764796.595226:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764796.595231:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764796.595235:0:64817:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764796.595242:0:64817:0:(vvp_io.c:826:vvp_io_read_start()) f25.sanity-pcc: read [21, 1048597) 00000001:00010000:2.0:1775764796.595254:0:64817:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x10:0x0] 00000008:00010000:2.0:1775764796.595257:0:64817:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764796.595258:0:64817:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c84b0, osc lock ffff8b238a3f8000, flags 0x400000400001002 00010000:00010000:2.0:1775764796.595261:0:64817:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764796.595266:0:64817:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:2.0:1775764796.595271:0:64817:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764796.595272:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764796.595277:0:64817:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:2.0:1775764796.595283:0:64817:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764796.595288:0:64817:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x10:0x0] updating i_size 21 i_blocks 1 00000020:00010000:2.0:1775764796.595290:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764796.595292:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764796.595294:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c84b0 (ffff8b23812ec5a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764796.595296:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764796.595298:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c84b0 (ffff8b23812eca28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764796.595303:0:64817:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:21 00000020:00010000:2.0:1775764796.595307:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764796.595310:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764796.595312:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8b30 (ffff8b23812ec5a8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764796.595315:0:64817:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764796.595324:0:64817:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000020:00010000:2.0:1775764796.595333:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36ba88 (ffff8b23812eca28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764796.595335:0:64817:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8b30 (ffff8b23812eca28/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764796.595339:0:64817:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1775764796.595353:0:64817:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764796.595356:0:64817:0:(file.c:2206:ll_file_io_generic()) f25.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1775764796.595358:0:64817:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:2.0:1775764796.595599:0:64817:0:(file.c:447:ll_file_release()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148), flags 100000 00000002:00100000:2.0:1775764796.595650:0:64817:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea4380 x1862022098649216/t34359739234(34359739234) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764812 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764796.595661:0:64817:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764796.595666:0:64817:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6a00 x1862022098649984/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764796.595687:0:64817:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64817:1862022098649984:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764796.595762:0:64817:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394fb40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764796.597974:0:64817:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64817:1862022098649984:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764796.598005:0:64817:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea4380 x1862022098649216/t34359739234(34359739234) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764812 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764796.691443:0:64827:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64827:1862022098650240:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764796.691492:0:64827:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17bfad lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64827 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764796.691517:0:64827:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17bfad lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64827 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764796.691525:0:64827:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17bfad lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64827 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764796.691532:0:64827:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17bfad lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64827 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764796.691539:0:64827:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17bfad lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64827 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764796.691545:0:64827:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2381497400 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764796.691549:0:64827:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17bfad lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64827 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764796.691556:0:64827:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17bfad lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64827 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764796.691568:0:64827:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17bfad lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 64827 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764796.691575:0:64827:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764796.691581:0:64827:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17bfad lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64827 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764796.691587:0:64827:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17bfad lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64827 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764796.691608:0:64827:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b2382daca80 x1862022098650240/t34359739237(34359739237) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764812 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764796.691620:0:64827:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382daca80 x1862022098650240/t34359739237(34359739237) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764812 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764796.691633:0:64827:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764796.691655:0:64827:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xf:0x0], updating i_size 512 00000002:00100000:1.0:1775764796.691677:0:64827:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382daca80 x1862022098650240/t34359739237(34359739237) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764812 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764796.691773:0:64827:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988) cmd=901066fc arg=7ffedb1839a0 00000100:00100000:2.0:1775764796.700979:0:64827:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64827:1862022098650368:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764796.701058:0:64827:0:(file.c:447:ll_file_release()) START file d25.sanity-pcc:[0x200003ab4:0xf:0x0](ffff8b2389a61988), flags 304000 00000002:00100000:2.0:1775764796.701114:0:64827:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b2382daca80 x1862022098650240/t34359739237(34359739237) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764812 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764796.701125:0:64827:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764796.701129:0:64827:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6a00 x1862022098650624/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764796.701150:0:64827:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64827:1862022098650624:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764796.701225:0:64827:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394fb40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764796.712678:0:64827:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64827:1862022098650624:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764796.712713:0:64827:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b2382daca80 x1862022098650240/t34359739237(34359739237) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764812 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764796.941055:0:64847:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764796.941079:0:64847:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764796.941095:0:64847:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764796.941107:0:64847:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764796.941116:0:64847:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764796.941122:0:64847:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764796.941125:0:64847:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:1.0:1775764796.941130:0:64847:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01e4bb58 00010000:00010000:1.0:1775764796.941133:0:64847:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764796.941139:0:64847:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764796.941149:0:64847:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764796.941154:0:64847:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d25.sanity-pcc, flags=25 00000080:00200000:1.0:1775764796.941158:0:64847:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764796.941161:0:64847:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f25.sanity-pcc, flags=9 00000080:00200008:1.0:1775764796.941165:0:64847:0:(file.c:6226:ll_getattr_dentry()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764796.941168:0:64847:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x10:0x0](ffff8b2390ec1148),name=f25.sanity-pcc 00000002:00010000:1.0:1775764796.941175:0:64847:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x10:0x0]) in obj [0x200003ab4:0xf:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764796.941180:0:64847:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764796.941185:0:64847:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764796.941189:0:64847:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200003ab4:0xf:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764796.941191:0:64847:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) 00000080:00010000:1.0:1775764796.941192:0:64847:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) for lock 0x73cfd86cfc17bfb4 00000080:00010000:1.0:1775764796.941194:0:64847:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bfb4 from it ffffa60b01e4bcb8 00010000:00010000:1.0:1775764796.941196:0:64847:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764796.941201:0:64847:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764796.941222:0:64847:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764796.941225:0:64847:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764796.941228:0:64847:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764796.941232:0:64847:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764796.941236:0:64847:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764796.941238:0:64847:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764796.941240:0:64847:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:1.0:1775764796.941241:0:64847:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01e4bbf8 00010000:00010000:1.0:1775764796.941243:0:64847:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764796.941247:0:64847:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764796.941251:0:64847:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764796.941253:0:64847:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d25.sanity-pcc, flags=25 00000080:00200000:1.0:1775764796.941254:0:64847:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764796.941257:0:64847:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f25.sanity-pcc, flags=265 00000080:00200000:1.0:1775764796.941259:0:64847:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x10:0x0](ffff8b2390ec1148), inode mode 81a4 mask 44 00000080:00200008:1.0:1775764796.941263:0:64847:0:(file.c:1057:ll_file_open()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148), flags 100000 00000002:00010000:1.0:1775764796.941272:0:64847:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x10:0x0]) in obj [0x200003ab4:0xf:0x0], intent: open flags 0240100001 00010000:00010000:1.0:1775764796.941324:0:64847:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17bfbb lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 64847 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764796.941335:0:64847:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17bfbb lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 64847 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764796.941346:0:64847:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764796.941360:0:64847:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fea00 x1862022098651392/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764796.941371:0:64847:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17bfbb lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 64847 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764796.941410:0:64847:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job mmap_cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:64847:1862022098651392:192.168.202.128@tcp:101:mmap_cat.0 00000100:00100000:1.0:1775764796.941943:0:64847:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b6543c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764796.946138:0:64847:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job mmap_cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:64847:1862022098651392:192.168.202.128@tcp:101:mmap_cat.0 00010000:00010000:1.0:1775764796.946160:0:64847:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17bfbb lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052556 expref: -99 pid: 64847 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764796.946173:0:64847:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17bfbb lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052556 expref: -99 pid: 64847 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764796.946179:0:64847:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17bfbb lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052556 expref: -99 pid: 64847 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764796.946190:0:64847:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17bfbb lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052556 expref: -99 pid: 64847 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764796.946198:0:64847:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17bfbb lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052556 expref: -99 pid: 64847 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764796.946207:0:64847:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17bfbb lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052556 expref: -99 pid: 64847 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764796.946212:0:64847:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17bfbb lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052556 expref: -99 pid: 64847 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764796.946217:0:64847:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17bfbb lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052556 expref: -99 pid: 64847 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764796.946223:0:64847:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23864fea00 x1862022098651392/t34359739241(34359739241) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764812 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'mmap_cat.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764796.946230:0:64847:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fea00 x1862022098651392/t34359739241(34359739241) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764812 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'mmap_cat.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764796.946239:0:64847:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17bfbb lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052556 expref: -99 pid: 64847 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764796.946440:0:64847:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17bfbb lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052556 expref: -99 pid: 64847 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.946558:0:64847:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.946572:0:64847:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.946579:0:64847:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17bfbb lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052556 expref: -99 pid: 64847 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.946585:0:64847:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17bfbb lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e03052556 expref: -99 pid: 64847 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.946596:0:64847:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17bfbb lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e03052556 expref: -99 pid: 64847 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764796.946649:0:64847:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:2.0:1775764796.946657:0:64847:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab4:0xf:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764796.946736:0:64847:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x10:0x0]: layout version change: 2 -> 2 00000080:00010000:2.0:1775764796.946743:0:64847:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) for lock 0x73cfd86cfc17bfb4 00000080:00010000:2.0:1775764796.946748:0:64847:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bfb4 from it ffffa60b01e4bbf0 00010000:00010000:2.0:1775764796.946750:0:64847:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764796.946756:0:64847:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000002:00100000:2.0:1775764796.946775:0:64847:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fea00 x1862022098651392/t34359739241(34359739241) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764812 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'mmap_cat.0' uid:0 gid:0 projid:0 00000080:00a00000:2.0:1775764796.946867:0:64847:0:(llite_mmap.c:586:ll_file_mmap()) VFS_Op: fid=[0x200003ab4:0x10:0x0] vma=ffff8b23865c2d98 start=0x7f51c5294000 end=0x7f51c5295000 vm_flags=0x71 00000080:02000000:2.0:1775764796.947331:0:64847:0:(pcc.c:3285:pcc_fault()) *** cfs_fail_loc=1411, val=0*** 00000080:00200000:2.0:1775764796.949012:0:64847:0:(rw.c:1928:ll_readpage()) fast read pgno: 0 00000080:00200000:2.0:1775764796.949022:0:64847:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764796.949031:0:64847:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x10:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764796.949033:0:64847:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764796.949038:0:64847:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764796.949058:0:64847:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 0] 00000008:00010000:2.0:1775764796.949068:0:64847:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764796.949071:0:64847:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c84b0, osc lock ffff8b238a3f8850, flags 0x0 00010000:00010000:2.0:1775764796.949089:0:64847:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1775764796.949102:0:64847:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1775764796.949111:0:64847:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1775764796.949117:0:64847:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1775764796.949121:0:64847:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000001:00010000:2.0:1775764796.949135:0:64847:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x10:0x0] 00000008:00010000:2.0:1775764796.949138:0:64847:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764796.949139:0:64847:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8c30, osc lock ffff8b238a3f8000, flags 0x400001002 00010000:00010000:2.0:1775764796.949143:0:64847:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1775764796.949149:0:64847:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000020:00200000:2.0:1775764796.949154:0:64847:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764796.949156:0:64847:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1775764796.949160:0:64847:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000020:00200000:2.0:1775764796.949164:0:64847:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764796.949169:0:64847:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x10:0x0] updating i_size 21 i_blocks 1 00000020:00010000:2.0:1775764796.949171:0:64847:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_release():245 00000020:00010000:2.0:1775764796.949174:0:64847:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764796.949175:0:64847:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8c30 (ffff8b23812ec5a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764796.949177:0:64847:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027d488 (ffff8b23812eca28/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764796.949179:0:64847:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8c30 (ffff8b23812eca28/1) at cl_lock_fini():72 00000008:00010000:2.0:1775764796.949216:0:64847:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b2383c9e540 00000008:00010000:2.0:1775764796.949265:0:46654:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b2383c9e540 00010000:00010000:2.0:1775764796.949282:0:46654:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000008:00100000:2.0:1775764796.949317:0:46654:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea6a00 - object 0x280000400:283 offset 0<>4096 00000100:00100000:2.0:1775764796.949390:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764796.949408:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098651520:192.168.202.128@tcp:3:lfs.0 00000100:00100000:1.0:1775764796.951684:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098651648:192.168.202.128@tcp:103:ldlm_bl.0 00000008:00010000:2.0:1775764796.954204:0:2379:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8b2383c9e540 00000020:00200000:2.0:1775764796.954220:0:2379:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1775764796.954788:0:2379:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8b2383c9e540 00000100:00100000:2.0:1775764796.954806:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098651520:192.168.202.128@tcp:3:lfs.0 00000080:00200000:1.0:1775764797.054702:0:64857:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764797.054740:0:64857:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764797.054761:0:64857:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764797.054778:0:64857:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764797.054787:0:64857:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764797.054798:0:64857:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764797.054802:0:64857:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:1.0:1775764797.054808:0:64857:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01db3bf8 00010000:00010000:1.0:1775764797.054812:0:64857:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764797.054820:0:64857:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:1.0:1775764797.054835:0:64857:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764797.054843:0:64857:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d25.sanity-pcc, flags=267 00000080:00200000:1.0:1775764797.054849:0:64857:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988), inode mode 41ed mask 44 00000080:00200008:1.0:1775764797.054855:0:64857:0:(file.c:1057:ll_file_open()) START file d25.sanity-pcc:[0x200003ab4:0xf:0x0](ffff8b2389a61988), flags 304000 00000002:00010000:1.0:1775764797.054870:0:64857:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xf:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:1.0:1775764797.054920:0:64857:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bfc2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764797.054927:0:64857:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bfc2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64857 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764797.054950:0:64857:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764797.054962:0:64857:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd180 x1862022098652160/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764797.054971:0:64857:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bfc2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64857 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764797.054995:0:64857:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64857:1862022098652160:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764797.055173:0:64857:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b6543c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764797.073109:0:64857:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64857:1862022098652160:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764797.073135:0:64857:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bfc2 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.073161:0:64857:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bfc2 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.073172:0:64857:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bfc2 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.073179:0:64857:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bfc2 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.073187:0:64857:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bfc2 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.073192:0:64857:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8741600 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764797.073197:0:64857:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bfc2 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.073204:0:64857:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bfc2 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.073211:0:64857:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bfc2 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 64857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.073218:0:64857:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764797.073227:0:64857:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bfc2 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64857 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.073234:0:64857:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17bfc2 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64857 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764797.073258:0:64857:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23864fd180 x1862022098652160/t34359739243(34359739243) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764813 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764797.073271:0:64857:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fd180 x1862022098652160/t34359739243(34359739243) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764813 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764797.073285:0:64857:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764797.073309:0:64857:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xf:0x0], updating i_size 512 00000002:00100000:2.0:1775764797.073342:0:64857:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fd180 x1862022098652160/t34359739243(34359739243) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764813 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764797.073448:0:64857:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988) cmd=901066fc arg=7fff5d860970 00000080:00200008:2.0:1775764797.073550:0:64857:0:(file.c:447:ll_file_release()) START file d25.sanity-pcc:[0x200003ab4:0xf:0x0](ffff8b2389a61988), flags 304000 00000002:00100000:2.0:1775764797.073606:0:64857:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fd180 x1862022098652160/t34359739243(34359739243) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764813 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764797.073615:0:64857:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764797.073620:0:64857:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6a00 x1862022098652416/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764797.073642:0:64857:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64857:1862022098652416:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764797.073708:0:64857:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764797.086841:0:64857:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64857:1862022098652416:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764797.086878:0:64857:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fd180 x1862022098652160/t34359739243(34359739243) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764813 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764797.222337:0:64864:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764797.222367:0:64864:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764797.223288:0:64864:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:1.0:1775764797.223307:0:64864:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:1.0:1775764797.223328:0:64864:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764797.223338:0:64864:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764797.223342:0:64864:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:1.0:1775764797.223350:0:64864:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01df3bf8 00010000:00010000:1.0:1775764797.223353:0:64864:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:1.0:1775764797.223363:0:64864:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:1.0:1775764797.223384:0:64864:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764797.223393:0:64864:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d25.sanity-pcc, flags=25 00000080:00200000:1.0:1775764797.223399:0:64864:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988), inode mode 41ed mask 1 00000080:00200000:1.0:1775764797.223403:0:64864:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f25.sanity-pcc, flags=265 00000080:00200000:1.0:1775764797.223407:0:64864:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x10:0x0](ffff8b2390ec1148), inode mode 81a4 mask 44 00000080:00200008:1.0:1775764797.223413:0:64864:0:(file.c:1057:ll_file_open()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148), flags 100000 00000002:00010000:1.0:1775764797.223430:0:64864:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x10:0x0]) in obj [0x200003ab4:0xf:0x0], intent: open flags 04240100001 00010000:00010000:1.0:1775764797.224292:0:64864:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 64864 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764797.224304:0:64864:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 64864 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764797.224317:0:64864:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764797.224329:0:64864:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fc000 x1862022098652672/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764797.224339:0:64864:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 64864 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764797.224360:0:64864:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:64864:1862022098652672:192.168.202.128@tcp:101:cat.0 00000100:00100000:1.0:1775764797.224913:0:64864:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764797.242182:0:64864:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:64864:1862022098652672:192.168.202.128@tcp:101:cat.0 00010000:00010000:2.0:1775764797.242289:0:64864:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052579 expref: -99 pid: 64864 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.242301:0:64864:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052579 expref: -99 pid: 64864 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.242318:0:64864:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052579 expref: -99 pid: 64864 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.242328:0:64864:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052579 expref: -99 pid: 64864 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764797.242342:0:64864:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23864fc000 x1862022098652672/t34359739245(34359739245) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764813 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764797.242353:0:64864:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fc000 x1862022098652672/t34359739245(34359739245) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764813 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764797.242365:0:64864:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052579 expref: -99 pid: 64864 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764797.242377:0:64864:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052579 expref: -99 pid: 64864 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.242387:0:64864:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/16 (0 0) 00000080:00200000:2.0:1775764797.242394:0:64864:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab4:0xf:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764797.242628:0:64864:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x10:0x0]: layout version change: 2 -> 2 00000080:00010000:2.0:1775764797.242641:0:64864:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) for lock 0x73cfd86cfc17bfc9 00000080:00010000:2.0:1775764797.242650:0:64864:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bfc9 from it ffffa60b01df3bf0 00010000:00010000:2.0:1775764797.242654:0:64864:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052579 expref: -99 pid: 64864 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.242664:0:64864:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052579 expref: -99 pid: 64864 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00100000:2.0:1775764797.242693:0:64864:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fc000 x1862022098652672/t34359739245(34359739245) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764813 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1775764797.242818:0:64864:0:(file.c:6226:ll_getattr_dentry()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764797.242826:0:64864:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x10:0x0](ffff8b2390ec1148),name=f25.sanity-pcc 00000002:00010000:2.0:1775764797.242845:0:64864:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x10:0x0]) in obj [0x200003ab4:0xf:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764797.242856:0:64864:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764797.242867:0:64864:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764797.242873:0:64864:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200003ab4:0xf:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764797.242879:0:64864:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) 00000080:00010000:2.0:1775764797.242882:0:64864:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) for lock 0x73cfd86cfc17bfb4 00000080:00010000:2.0:1775764797.242885:0:64864:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bfb4 from it ffffa60b01df3cd8 00010000:00010000:2.0:1775764797.242888:0:64864:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764797.242896:0:64864:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764797.246008:0:64864:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:2.0:1775764797.246026:0:64864:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x10:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764797.246030:0:64864:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1775764797.246033:0:64864:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x10:0x0] 00000008:00010000:2.0:1775764797.246073:0:64864:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764797.246077:0:64864:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8c30, osc lock ffff8b238a3f8000, flags 0x400000400001002 00010000:00010000:2.0:1775764797.246101:0:64864:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1775764797.246119:0:64864:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000020:00200000:2.0:1775764797.246133:0:64864:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764797.246137:0:64864:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1775764797.246147:0:64864:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00200000:2.0:1775764797.246162:0:64864:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764797.246172:0:64864:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x10:0x0] updating i_size 21 i_blocks 1 00000020:00010000:2.0:1775764797.246176:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c920 (ffff8b23812ec008/0) at cl_lock_release():245 00000020:00010000:2.0:1775764797.246179:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c920 (ffff8b23812ec008/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764797.246183:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8c30 (ffff8b23812ec3f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764797.246186:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c920 (ffff8b23812ec008/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764797.246189:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8c30 (ffff8b23812ec008/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764797.246220:0:64864:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/1, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1775764797.246319:0:64864:0:(file.c:2454:do_file_read_iter()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0], ppos: 0, count: 4194304 00000080:00200000:2.0:1775764797.246363:0:64864:0:(file.c:2058:ll_file_io_generic()) f25.sanity-pcc: read ppos: 21, bytes: 4194283 00000080:00200000:2.0:1775764797.246368:0:64864:0:(file.c:1815:ll_io_set_mirror()) f25.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764797.246374:0:64864:0:(cl_io.c:214:cl_io_rw_init()) header@00000000ab8aad76[0x110, 2, [0x200003ab4:0x10:0x0] hash] 00000020:00200000:2.0:1775764797.246376:0:64864:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [21, 4194304) 0 0 00000080:00200000:2.0:1775764797.246379:0:64864:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764797.246387:0:64864:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x10:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764797.246389:0:64864:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764797.246396:0:64864:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 21 chunk: [21, 4194304) 4194304, 4194283 00020000:00200000:2.0:1775764797.246399:0:64864:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764797.246418:0:64864:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 1023] 00000008:00010000:2.0:1775764797.246426:0:64864:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764797.246429:0:64864:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8c30, osc lock ffff8b238a3f8000, flags 0x400000000000000 00010000:00010000:2.0:1775764797.246438:0:64864:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764797.246448:0:64864:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4194303) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764797.246484:0:64864:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764797.246492:0:64864:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764797.246498:0:64864:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000080:00200000:2.0:1775764797.246507:0:64864:0:(vvp_io.c:826:vvp_io_read_start()) f25.sanity-pcc: read [21, 4194304) 00000001:00010000:2.0:1775764797.246518:0:64864:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x10:0x0] 00000008:00010000:2.0:1775764797.246523:0:64864:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764797.246525:0:64864:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c86b0, osc lock ffff8b238a3f8be0, flags 0x400000400001002 00010000:00010000:2.0:1775764797.246532:0:64864:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764797.246546:0:64864:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00200000:2.0:1775764797.246554:0:64864:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764797.246556:0:64864:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764797.246563:0:64864:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00200000:2.0:1775764797.246569:0:64864:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764797.246574:0:64864:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x10:0x0] updating i_size 21 i_blocks 1 00000020:00010000:2.0:1775764797.246576:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c920 (ffff8b23812ec008/0) at cl_lock_release():245 00000020:00010000:2.0:1775764797.246579:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c920 (ffff8b23812ec008/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764797.246581:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c86b0 (ffff8b23812ec3f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764797.246584:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c920 (ffff8b23812ec008/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764797.246587:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c86b0 (ffff8b23812ec008/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764797.246592:0:64864:0:(vvp_io.c:885:vvp_io_read_start()) count:4194283 iocb pos:21 00000020:00010000:2.0:1775764797.246600:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b8a8 (ffff8b23812ec008/0) at cl_lock_release():245 00000020:00010000:2.0:1775764797.246602:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b8a8 (ffff8b23812ec008/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764797.246604:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8c30 (ffff8b23812ec3f8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764797.246608:0:64864:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764797.246616:0:64864:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000020:00010000:2.0:1775764797.246625:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b8a8 (ffff8b23812ec008/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764797.246628:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8c30 (ffff8b23812ec008/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764797.246632:0:64864:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1775764797.246645:0:64864:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764797.246649:0:64864:0:(file.c:2206:ll_file_io_generic()) f25.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1775764797.246652:0:64864:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:2.0:1775764797.246688:0:64864:0:(file.c:2454:do_file_read_iter()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0], ppos: 21, count: 4194304 00000080:00200000:2.0:1775764797.246703:0:64864:0:(file.c:2058:ll_file_io_generic()) f25.sanity-pcc: read ppos: 21, bytes: 4194304 00000080:00200000:2.0:1775764797.246705:0:64864:0:(file.c:1815:ll_io_set_mirror()) f25.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764797.246709:0:64864:0:(cl_io.c:214:cl_io_rw_init()) header@00000000ab8aad76[0x110, 2, [0x200003ab4:0x10:0x0] hash] 00000020:00200000:2.0:1775764797.246711:0:64864:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [21, 4194325) 0 0 00000080:00200000:2.0:1775764797.246713:0:64864:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764797.246719:0:64864:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x10:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764797.246721:0:64864:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764797.246724:0:64864:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 21 chunk: [21, 4194325) 4194325, 4194304 00020000:00200000:2.0:1775764797.246727:0:64864:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764797.246737:0:64864:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 1024] 00000008:00010000:2.0:1775764797.246742:0:64864:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764797.246745:0:64864:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8c30, osc lock ffff8b238a3f8000, flags 0x400000000000000 00010000:00010000:2.0:1775764797.246752:0:64864:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1775764797.246761:0:64864:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4198399) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1775764797.246770:0:64864:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1775764797.246784:0:64864:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1775764797.246791:0:64864:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000080:00200000:2.0:1775764797.246797:0:64864:0:(vvp_io.c:826:vvp_io_read_start()) f25.sanity-pcc: read [21, 4194325) 00000001:00010000:2.0:1775764797.246803:0:64864:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x10:0x0] 00000008:00010000:2.0:1775764797.246806:0:64864:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764797.246808:0:64864:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c86b0, osc lock ffff8b238a3f8be0, flags 0x400000400001002 00010000:00010000:2.0:1775764797.246812:0:64864:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1775764797.246819:0:64864:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000020:00200000:2.0:1775764797.246827:0:64864:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764797.246830:0:64864:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1775764797.246837:0:64864:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000020:00200000:2.0:1775764797.246843:0:64864:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764797.246846:0:64864:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x10:0x0] updating i_size 21 i_blocks 1 00000020:00010000:2.0:1775764797.246848:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c920 (ffff8b23812ec008/0) at cl_lock_release():245 00000020:00010000:2.0:1775764797.246851:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c920 (ffff8b23812ec008/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764797.246853:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c86b0 (ffff8b23812ec3f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764797.246856:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c920 (ffff8b23812ec008/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764797.246858:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c86b0 (ffff8b23812ec008/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764797.246862:0:64864:0:(vvp_io.c:885:vvp_io_read_start()) count:4194304 iocb pos:21 00000020:00010000:2.0:1775764797.246865:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b8a8 (ffff8b23812ec008/0) at cl_lock_release():245 00000020:00010000:2.0:1775764797.246867:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b8a8 (ffff8b23812ec008/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764797.246870:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8c30 (ffff8b23812ec3f8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764797.246873:0:64864:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1775764797.246881:0:64864:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00000020:00010000:2.0:1775764797.246889:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b8a8 (ffff8b23812ec008/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764797.246891:0:64864:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8c30 (ffff8b23812ec008/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764797.246894:0:64864:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1775764797.246904:0:64864:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764797.246907:0:64864:0:(file.c:2206:ll_file_io_generic()) f25.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1775764797.246909:0:64864:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:2.0:1775764797.246995:0:64864:0:(file.c:447:ll_file_release()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148), flags 100000 00010000:00010000:2.0:1775764797.247013:0:64864:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052579 expref: -99 pid: 64864 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000100:00100000:1.0:1775764797.331570:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764797.333650:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764797.333662:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764797.333679:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5ea00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764797.333684:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764797.333692:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764797.333701:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764797.333730:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052579 expref: -99 pid: 64864 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764797.333739:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764797.333793:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764797.333800:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764797.333839:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764797.333847:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bfb4 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052541 expref: -99 pid: 64834 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764797.333865:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764797.333867:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764797.333869:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1775764797.467649:0:64880:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764797.467677:0:64880:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764797.467698:0:64880:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1775764797.467715:0:64880:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1775764797.467724:0:64880:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764797.467732:0:64880:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764797.467735:0:64880:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:2.0:1775764797.467742:0:64880:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01eb3bf8 00010000:00010000:2.0:1775764797.467746:0:64880:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1775764797.467754:0:64880:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1775764797.467770:0:64880:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764797.467778:0:64880:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d25.sanity-pcc, flags=267 00000080:00200000:2.0:1775764797.467785:0:64880:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988), inode mode 41ed mask 44 00000080:00200008:2.0:1775764797.467792:0:64880:0:(file.c:1057:ll_file_open()) START file d25.sanity-pcc:[0x200003ab4:0xf:0x0](ffff8b2389a61988), flags 304000 00000002:00010000:2.0:1775764797.467808:0:64880:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xf:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:2.0:1775764797.467959:0:64880:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfd7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64880 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.467968:0:64880:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfd7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64880 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764797.467983:0:64880:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764797.468000:0:64880:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4380 x1862022098653312/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764797.468010:0:64880:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfd7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64880 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764797.468038:0:64880:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64880:1862022098653312:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764797.468534:0:64880:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764797.473214:0:64880:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64880:1862022098653312:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764797.473239:0:64880:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfd7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64880 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.473253:0:64880:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfd7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64880 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.473261:0:64880:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfd7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64880 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.473280:0:64880:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfd7 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64880 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.473291:0:64880:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfd7 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64880 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.473297:0:64880:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2380030400 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764797.473303:0:64880:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfd7 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64880 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.473315:0:64880:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfd7 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64880 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.473323:0:64880:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfd7 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 64880 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.473331:0:64880:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764797.473337:0:64880:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfd7 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64880 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.473344:0:64880:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfd7 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64880 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764797.473376:0:64880:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23b9ea4380 x1862022098653312/t34359739247(34359739247) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764813 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764797.473390:0:64880:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4380 x1862022098653312/t34359739247(34359739247) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764813 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764797.473402:0:64880:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764797.473423:0:64880:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xf:0x0], updating i_size 512 00000002:00100000:2.0:1775764797.473443:0:64880:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4380 x1862022098653312/t34359739247(34359739247) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764813 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764797.473600:0:64880:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988) cmd=901066fc arg=7ffdb76c6240 00000100:00100000:2.0:1775764797.473664:0:64880:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64880:1862022098653440:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764797.473741:0:64880:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764797.483861:0:64880:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64880:1862022098653440:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764797.483946:0:64880:0:(file.c:447:ll_file_release()) START file d25.sanity-pcc:[0x200003ab4:0xf:0x0](ffff8b2389a61988), flags 304000 00000002:00100000:2.0:1775764797.484006:0:64880:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea4380 x1862022098653312/t34359739247(34359739247) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764813 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764797.484018:0:64880:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764797.484023:0:64880:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6a00 x1862022098653696/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764797.484044:0:64880:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64880:1862022098653696:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764797.484134:0:64880:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764797.487995:0:64880:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64880:1862022098653696:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764797.488030:0:64880:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea4380 x1862022098653312/t34359739247(34359739247) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764813 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764797.576217:0:64571:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764797.576244:0:64571:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764797.576263:0:64571:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1775764797.576281:0:64571:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1775764797.576291:0:64571:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764797.576300:0:64571:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764797.576304:0:64571:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:2.0:1775764797.576313:0:64571:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01c0bbf8 00010000:00010000:2.0:1775764797.576316:0:64571:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1775764797.576324:0:64571:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:2.0:1775764797.576342:0:64571:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764797.576350:0:64571:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d25.sanity-pcc, flags=25 00000080:00200000:2.0:1775764797.576357:0:64571:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764797.576372:0:64571:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988), inode mode 41ed mask 3 00000080:00200000:2.0:1775764797.576376:0:64571:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f25.sanity-pcc, dir=[0x200003ab4:0xf:0x0](ffff8b2389a61988), file ffff8b2383ef1b00, open_flags 8241, mode 81b6 opened 0 00000080:00200000:2.0:1775764797.576385:0:64571:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f25.sanity-pcc, dir=[0x200003ab4:0xf:0x0](ffff8b2389a61988), intent=open|creat 00000002:00010000:2.0:1775764797.576406:0:64571:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f25.sanity-pcc,[0x200003ab4:0x11:0x0]) in obj [0x200003ab4:0xf:0x0], intent: open|creat flags 0101102 00010000:00010000:2.0:1775764797.576420:0:64571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf98 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e030524d8 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764797.576438:0:64571:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf83 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030524a0 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764797.576513:0:64571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf98 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030524d8 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764797.576520:0:64571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764797.576523:0:64571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea200/0x73cfd86cfc17bf98 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030524d8 expref: -99 pid: 64817 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764797.576551:0:64571:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 3/0,1 mode: --/CW res: [0x200003ab4:0xf:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64571 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.576558:0:64571:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 3/0,1 mode: --/CW res: [0x200003ab4:0xf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64571 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764797.576569:0:64571:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764797.576581:0:64571:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6a00 x1862022098653952/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764797.576590:0:64571:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 3/0,1 mode: --/CW res: [0x200003ab4:0xf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64571 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764797.576611:0:64571:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098653952:192.168.202.128@tcp:101:bash.0 00000100:00100000:2.0:1775764797.576766:0:64571:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764797.588959:0:64571:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098653952:192.168.202.128@tcp:101:bash.0 00010000:00010000:2.0:1775764797.588989:0:64571:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 4/0,1 mode: --/CW res: [0x200003ab4:0xf:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030525d4 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.589009:0:64571:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030525d4 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.589018:0:64571:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030525d4 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.589031:0:64571:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030525d4 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.589041:0:64571:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030525d4 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.589054:0:64571:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030525d4 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.589063:0:64571:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030525d4 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.589070:0:64571:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030525d4 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764797.589081:0:64571:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=2b, status=0 req@ffff8b23b9ea6a00 x1862022098653952/t34359739249(34359739249) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/864 e 0 to 0 dl 1775764813 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764797.589094:0:64571:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea6a00 x1862022098653952/t34359739249(34359739249) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/864 e 0 to 0 dl 1775764813 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764797.589107:0:64571:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030525d4 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764797.589121:0:64571:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030525d4 expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.589131:0:64571:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/16 (0 0) 00000080:00200000:2.0:1775764797.589137:0:64571:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200003ab4:0xf:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764797.589236:0:64571:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0x10:0x0](ffff8b2390ec1148): rc = 0 00000080:00200000:2.0:1775764797.589257:0:64571:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x10:0x0]: layout version change: 2 -> 2 00000080:00010000:2.0:1775764797.589268:0:64571:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) for lock 0x73cfd86cfc17bfde 00000080:00010000:2.0:1775764797.589281:0:64571:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) 00000080:00200008:2.0:1775764797.589296:0:64571:0:(file.c:1057:ll_file_open()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148), flags 101101 00000002:00100000:2.0:1775764797.589332:0:64571:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea6a00 x1862022098653952/t34359739249(34359739249) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764813 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764797.589371:0:64571:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea6a00 x1862022098653952/t34359739249(34359739249) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764813 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764797.589380:0:64571:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764797.589385:0:64571:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4380 x1862022098654208/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764797.589405:0:64571:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098654208:192.168.202.128@tcp:35:bash.0 00000100:00100000:2.0:1775764797.589522:0:64571:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764797.606869:0:64571:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098654208:192.168.202.128@tcp:35:bash.0 00000002:00100000:2.0:1775764797.606905:0:64571:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea6a00 x1862022098653952/t34359739249(34359739249) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764813 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764797.606977:0:64571:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bfde from it ffff8b238504c9c0 00010000:00010000:2.0:1775764797.606988:0:64571:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030525d4 expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.607002:0:64571:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030525d4 expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764797.607024:0:64571:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x10:0x0](ffff8b2390ec1148), inode mode 81a4 mask 42 00000080:00200008:2.0:1775764797.607040:0:64571:0:(llite_lib.c:2274:ll_setattr_raw()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148) current size 21, valid attrs a068, mode 0, uid 2, gid 0, new size 0, atime 1775764797.000000000, mtime 1775764797.000000000, ctime 1775764797.000000000 00010000:00010000:2.0:1775764797.607070:0:64571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1a/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e030525d4 expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764797.607090:0:64571:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052579 expref: -99 pid: 64864 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764797.607106:0:64571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764797.609397:0:64571:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052579 expref: -99 pid: 64864 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764797.609489:0:64571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030525d4 expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764797.609498:0:64571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764797.609504:0:64571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1775764797.609507:0:64571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17bfde lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030525d4 expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764797.609524:0:64571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfa6 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052502 expref: -99 pid: 64817 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000020:00100000:2.0:1775764797.609539:0:64571:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764797.609551:0:64571:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6a00 x1862022098654464/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764797.609575:0:64571:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098654464:192.168.202.128@tcp:36:bash.0 00000100:00100000:2.0:1775764797.609682:0:64571:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764797.612765:0:64571:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098654464:192.168.202.128@tcp:36:bash.0 00000080:00200000:2.0:1775764797.612847:0:64571:0:(file.c:1815:ll_io_set_mirror()) f25.sanity-pcc: desiginated mirror: 0 00000080:00200000:2.0:1775764797.612853:0:64571:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764797.612865:0:64571:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764797.612870:0:64571:0:(vvp_io.c:355:vvp_io_fini()) [0x200003ab4:0x10:0x0] write layout, type 3 [0x0, 0x0) 00000080:00010000:2.0:1775764797.612878:0:64571:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x10:0x0](ffff8b2390ec1148) 00000002:00010000:2.0:1775764797.612889:0:64571:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x10:0x0]) in obj [0x200003ab4:0x10:0x0], intent: layout flags 02 00010000:00010000:2.0:1775764797.612909:0:64571:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052579 expref: -99 pid: 64864 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000002:00100000:2.0:1775764797.612969:0:64571:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fc000 x1862022098652672/t34359739245(34359739245) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764813 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cat.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764797.612980:0:64571:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764797.612985:0:64571:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4e00 x1862022098654720/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764797.613005:0:64571:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098654720:192.168.202.128@tcp:35:bash.0 00000100:00100000:2.0:1775764797.613057:0:64571:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764797.616763:0:64571:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098654720:192.168.202.128@tcp:35:bash.0 00000002:00100000:2.0:1775764797.616798:0:64571:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fc000 x1862022098652672/t34359739245(34359739245) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764813 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cat.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764797.616949:0:64571:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17bfd0 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x10:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305258e expref: -99 pid: 64870 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764797.616997:0:64571:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x10:0x0]: losing layout lock 00010000:00010000:2.0:1775764797.617058:0:64571:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052579 expref: -99 pid: 64864 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764797.617066:0:64571:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764797.617069:0:64571:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17bfc9 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052579 expref: -99 pid: 64864 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764797.617101:0:64571:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfe5 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x10:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64571 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.617110:0:64571:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfe5 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x10:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64571 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764797.617119:0:64571:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764797.617126:0:64571:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4380 x1862022098654976/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/880 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764797.617136:0:64571:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfe5 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x10:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64571 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764797.617155:0:64571:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098654976:192.168.202.128@tcp:101:bash.0 00000100:00100000:2.0:1775764797.617296:0:64571:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764797.627937:0:64571:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:64571:1862022098654976:192.168.202.128@tcp:101:bash.0 00010000:00010000:2.0:1775764797.627968:0:64571:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfe5 lrc: 4/0,1 mode: --/EX res: [0x200003ab4:0x10:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030525f0 expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.627981:0:64571:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfe5 lrc: 4/0,1 mode: --/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030525f0 expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.627991:0:64571:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfe5 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030525f0 expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.628002:0:64571:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfe5 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030525f0 expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.628018:0:64571:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfe5 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030525f0 expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.628026:0:64571:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfe5 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030525f0 expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.628033:0:64571:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfe5 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030525f0 expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764797.628044:0:64571:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea4380 x1862022098654976/t34359739254(34359739254) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/600 e 0 to 0 dl 1775764813 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764797.628067:0:64571:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfe5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030525f0 expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764797.628081:0:64571:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfe5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030525f0 expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.628096:0:64571:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/16 (0 0) 00000080:00200000:2.0:1775764797.628103:0:64571:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x10:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764797.628116:0:64571:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) for lock 0x73cfd86cfc17bfe5 00000080:00010000:2.0:1775764797.628124:0:64571:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bfe5 from it ffffa60b01c0b930 00010000:00010000:2.0:1775764797.628127:0:64571:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfe5 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030525f0 expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764797.628137:0:64571:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17bfe5 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030525f0 expref: -99 pid: 64571 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764797.628160:0:64571:0:(file.c:1815:ll_io_set_mirror()) f25.sanity-pcc: desiginated mirror: 0 00000080:00200000:2.0:1775764797.628166:0:64571:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 2 restore needed 0 00000100:00100000:1.0:1775764797.868087:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764797.869029:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764797.869036:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17bfd0 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x10:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e0305258e expref: -99 pid: 64870 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764797.869046:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2381497800 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764797.869052:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17bfd0 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x10:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e0305258e expref: -99 pid: 64870 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764797.869062:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17bfd0 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x10:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e0305258e expref: -99 pid: 64870 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764797.869072:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17bfd0 lrc: 3/0,0 mode: CW/CW res: [0x200003ab4:0x10:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x57b8ff0e0305258e expref: -99 pid: 64870 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000002:00100000:2.0:1775764797.869144:0:46149:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b2382dad500 x1862022098652928/t34359739246(34359739246) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764813 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764797.869155:0:46149:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764797.869160:0:46149:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4e00 x1862022098655872/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764797.869184:0:46149:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:46149:1862022098655872:192.168.202.128@tcp:35:ldlm_bl.0 00000100:00100000:2.0:1775764797.869251:0:46149:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764797.875206:0:46149:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:46149:1862022098655872:192.168.202.128@tcp:35:ldlm_bl.0 00000002:00100000:2.0:1775764797.875253:0:46149:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b2382dad500 x1862022098652928/t34359739246(34359739246) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764813 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764797.875351:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17bfd0 lrc: 2/0,0 mode: --/CW res: [0x200003ab4:0x10:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e0305258e expref: -99 pid: 64870 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764797.875361:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764797.875422:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17bfd0 lrc: 1/0,0 mode: --/CW res: [0x200003ab4:0x10:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e0305258e expref: -99 pid: 64870 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764797.875431:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17bfd0 lrc: 0/0,0 mode: --/CW res: [0x200003ab4:0x10:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e0305258e expref: -99 pid: 64870 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764797.875448:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764797.875450:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764797.875491:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764797.875508:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764797.875525:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098656000:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764797.894448:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098656000:192.168.202.128@tcp:103:ldlm_bl.0 00000080:00200000:1.0:1775764797.942006:0:64892:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1775764797.942104:0:64892:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c008 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052652 expref: -99 pid: 64892 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764797.942119:0:64892:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c008 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052652 expref: -99 pid: 64892 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1775764797.942131:0:64892:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x10:0x0](000000007b174fdb) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c008 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052652 expref: -99 pid: 64892 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764797.942198:0:64892:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764797.942207:0:64892:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x10:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764797.942210:0:64892:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1775764797.942215:0:64892:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764797.942255:0:64892:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1775764797.942300:0:64892:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 21 blocks 1 [cma]time: 1775764796 1775764796 1775764797 by oinfo size 16 blocks 1 [cma]time 1775764797 1775764797 1775764797 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1775764797.942370:0:64892:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x10:0x0]: layout version change: 4294967294 -> 5 00010000:00010000:1.0:1775764797.942385:0:64892:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c008 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052652 expref: -99 pid: 64892 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764797.942394:0:64892:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c008 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052652 expref: -99 pid: 64892 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:1.0:1775764797.942907:0:64892:0:(file.c:2454:do_file_read_iter()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0], ppos: 0, count: 1048576 00000080:00200000:1.0:1775764797.942939:0:64892:0:(rw.c:1928:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1775764797.942959:0:64892:0:(file.c:2058:ll_file_io_generic()) f25.sanity-pcc: read ppos: 0, bytes: 1048576 00000080:00200000:1.0:1775764797.942963:0:64892:0:(file.c:1815:ll_io_set_mirror()) f25.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764797.942969:0:64892:0:(cl_io.c:214:cl_io_rw_init()) header@00000000ab8aad76[0x110, 1, [0x200003ab4:0x10:0x0] hash] 00000020:00200000:1.0:1775764797.942971:0:64892:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 1048576) 0 0 00000080:00200000:1.0:1775764797.942974:0:64892:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764797.942984:0:64892:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x10:0x0]: flr state: 10, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764797.942987:0:64892:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:1.0:1775764797.942994:0:64892:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:1.0:1775764797.942997:0:64892:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764797.943016:0:64892:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:1.0:1775764797.943029:0:64892:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764797.943033:0:64892:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c59b0, osc lock ffff8b2382cf6130, flags 0x400000000000000 00010000:00010000:1.0:1775764797.943052:0:64892:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764797.943064:0:64892:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764797.943076:0:64892:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00000008:00010000:1.0:1775764797.943086:0:64892:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 16, setting rss=16; kms=16, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 5/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764797.943097:0:64892:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764797.943105:0:64892:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1775764797.943114:0:64892:0:(vvp_io.c:826:vvp_io_read_start()) f25.sanity-pcc: read [0, 1048576) 00000001:00010000:1.0:1775764797.943129:0:64892:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x10:0x0] 00000008:00010000:1.0:1775764797.943134:0:64892:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764797.943136:0:64892:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5930, osc lock ffff8b2382cf6e40, flags 0x400000400001002 00010000:00010000:1.0:1775764797.943143:0:64892:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764797.943151:0:64892:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00000020:00200000:1.0:1775764797.943159:0:64892:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764797.943162:0:64892:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1775764797.943169:0:64892:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00000020:00200000:1.0:1775764797.943175:0:64892:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764797.943181:0:64892:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x10:0x0] updating i_size 16 i_blocks 1 00000020:00010000:1.0:1775764797.943184:0:64892:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_release():245 00000020:00010000:1.0:1775764797.943187:0:64892:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764797.943190:0:64892:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5930 (ffff8b2387cba008/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764797.943194:0:64892:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f19918 (ffff8b2387cba368/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764797.943196:0:64892:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5930 (ffff8b2387cba368/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764797.943203:0:64892:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:0 00000008:00010000:1.0:1775764797.943241:0:64892:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b238a0ce000 00000008:00010000:1.0:1775764797.943307:0:54744:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b238a0ce000 00010000:00010000:1.0:1775764797.943327:0:54744:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00000008:00100000:1.0:1775764797.943368:0:54744:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23864fdc00 - object 0x280000400:283 offset 0<>4096 00000100:00100000:1.0:1775764797.943415:0:2377:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764797.943433:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098656768:192.168.202.128@tcp:3:lfs.0 00000008:00010000:1.0:1775764797.949151:0:2377:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8b238a0ce000 00000020:00200000:1.0:1775764797.949166:0:2377:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1775764797.949243:0:2377:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8b238a0ce000 00000100:00100000:1.0:1775764797.949287:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098656768:192.168.202.128@tcp:3:lfs.0 00000080:00200008:2.0:1775764797.953110:0:64892:0:(file.c:447:ll_file_release()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148), flags 100000 00000002:00100000:2.0:1775764797.953177:0:64892:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864f3b80 x1862022098656256/t34359739257(34359739257) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764813 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764797.953189:0:64892:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764797.953194:0:64892:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4e00 x1862022098657024/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764797.953218:0:64892:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64892:1862022098657024:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764797.953319:0:64892:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764797.966533:0:64892:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64892:1862022098657024:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764797.966570:0:64892:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864f3b80 x1862022098656256/t34359739257(34359739257) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764813 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764798.096732:0:64902:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c00f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64902 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764798.096743:0:64902:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c00f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64902 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764798.096755:0:64902:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764798.096767:0:64902:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864f2d80 x1862022098657280/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764798.096778:0:64902:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c00f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64902 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764798.096801:0:64902:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64902:1862022098657280:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764798.097008:0:64902:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764798.101533:0:64902:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64902:1862022098657280:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764798.101562:0:64902:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c00f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64902 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764798.101576:0:64902:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c00f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64902 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764798.101583:0:64902:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c00f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64902 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764798.101591:0:64902:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c00f lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64902 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764798.101605:0:64902:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c00f lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64902 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764798.101611:0:64902:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2380030400 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764798.101616:0:64902:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c00f lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64902 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764798.101624:0:64902:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c00f lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64902 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764798.101631:0:64902:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c00f lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 64902 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764798.101638:0:64902:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1775764798.101644:0:64902:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c00f lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64902 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764798.101651:0:64902:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c00f lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64902 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764798.101672:0:64902:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23864f2d80 x1862022098657280/t34359739260(34359739260) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764814 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764798.101686:0:64902:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864f2d80 x1862022098657280/t34359739260(34359739260) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764814 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764798.101710:0:64902:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764798.101741:0:64902:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xf:0x0], updating i_size 7680 00000002:00100000:2.0:1775764798.101761:0:64902:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864f2d80 x1862022098657280/t34359739260(34359739260) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764814 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764798.101862:0:64902:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988) cmd=901066fc arg=7ffc2f82b3e0 00000100:00100000:2.0:1775764798.101947:0:64902:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64902:1862022098657408:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764798.102050:0:64902:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764798.118859:0:64902:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64902:1862022098657408:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764798.118938:0:64902:0:(file.c:447:ll_file_release()) START file d25.sanity-pcc:[0x200003ab4:0xf:0x0](ffff8b2389a61988), flags 304000 00000002:00100000:2.0:1775764798.118987:0:64902:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864f2d80 x1862022098657280/t34359739260(34359739260) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764814 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764798.118999:0:64902:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764798.119004:0:64902:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4e00 x1862022098657664/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764798.119026:0:64902:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64902:1862022098657664:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764798.119099:0:64902:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764798.136583:0:64902:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64902:1862022098657664:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764798.136619:0:64902:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864f2d80 x1862022098657280/t34359739260(34359739260) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764814 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764798.519416:0:64922:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:64922:1862022098658432:192.168.202.128@tcp:101:cat.0 00010000:00010000:1.0:1775764798.519547:0:64922:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17c01d lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030526a6 expref: -99 pid: 64922 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764798.519559:0:64922:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17c01d lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030526a6 expref: -99 pid: 64922 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764798.519570:0:64922:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17c01d lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030526a6 expref: -99 pid: 64922 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764798.519580:0:64922:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17c01d lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030526a6 expref: -99 pid: 64922 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764798.519596:0:64922:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b2382daea00 x1862022098658432/t34359739264(34359739264) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764814 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764798.519609:0:64922:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382daea00 x1862022098658432/t34359739264(34359739264) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764814 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764798.519620:0:64922:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17c01d lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030526a6 expref: -99 pid: 64922 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764798.519635:0:64922:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17c01d lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030526a6 expref: -99 pid: 64922 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764798.519646:0:64922:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/16 (0 0) 00000080:00200000:1.0:1775764798.519652:0:64922:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab4:0xf:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764798.519754:0:64922:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x10:0x0]: layout version change: 5 -> 5 00000080:00010000:1.0:1775764798.519764:0:64922:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) for lock 0x73cfd86cfc17c01d 00000080:00010000:1.0:1775764798.519771:0:64922:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c01d from it ffffa60b01ed3bf0 00010000:00010000:1.0:1775764798.519775:0:64922:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17c01d lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030526a6 expref: -99 pid: 64922 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764798.519783:0:64922:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17c01d lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030526a6 expref: -99 pid: 64922 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764798.519876:0:64922:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382daea00 x1862022098658432/t34359739264(34359739264) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764814 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1775764798.520061:0:64922:0:(file.c:6226:ll_getattr_dentry()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764798.520069:0:64922:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x10:0x0](ffff8b2390ec1148),name=f25.sanity-pcc 00000002:00010000:1.0:1775764798.520091:0:64922:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x10:0x0]) in obj [0x200003ab4:0xf:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764798.520101:0:64922:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c016 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052691 expref: -99 pid: 64909 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764798.520112:0:64922:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c016 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052691 expref: -99 pid: 64909 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764798.520120:0:64922:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200003ab4:0xf:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764798.520125:0:64922:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) 00000080:00010000:1.0:1775764798.520129:0:64922:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) for lock 0x73cfd86cfc17c016 00000080:00010000:1.0:1775764798.520133:0:64922:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c016 from it ffffa60b01ed3cd8 00010000:00010000:1.0:1775764798.520136:0:64922:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c016 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052691 expref: -99 pid: 64909 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764798.520147:0:64922:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c016 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052691 expref: -99 pid: 64909 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:1.0:1775764798.520250:0:64922:0:(file.c:2454:do_file_read_iter()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0], ppos: 0, count: 4194304 00000080:02000000:1.0:1775764798.520257:0:64922:0:(pcc.c:2487:pcc_file_read_iter()) *** cfs_fail_loc=1411, val=0*** 00000080:00200000:1.0:1775764798.524124:0:64922:0:(file.c:2058:ll_file_io_generic()) f25.sanity-pcc: read ppos: 16, bytes: 4194288 00000080:00200000:1.0:1775764798.524134:0:64922:0:(file.c:1815:ll_io_set_mirror()) f25.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764798.524142:0:64922:0:(cl_io.c:214:cl_io_rw_init()) header@00000000ab8aad76[0x110, 2, [0x200003ab4:0x10:0x0] hash] 00000020:00200000:1.0:1775764798.524144:0:64922:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [16, 4194304) 0 0 00000080:00200000:1.0:1775764798.524149:0:64922:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764798.524165:0:64922:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x10:0x0]: flr state: 10, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764798.524169:0:64922:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:1.0:1775764798.524177:0:64922:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 16 chunk: [16, 4194304) 4194304, 4194288 00020000:00200000:1.0:1775764798.524181:0:64922:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764798.524210:0:64922:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 1023] 00000008:00010000:1.0:1775764798.524224:0:64922:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764798.524229:0:64922:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5330, osc lock ffff8b2382cf6e40, flags 0x400000000000000 00010000:00010000:1.0:1775764798.524252:0:64922:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764798.524268:0:64922:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4194303) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764798.524281:0:64922:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764798.524289:0:64922:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764798.524301:0:64922:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1775764798.524310:0:64922:0:(vvp_io.c:826:vvp_io_read_start()) f25.sanity-pcc: read [16, 4194304) 00000001:00010000:1.0:1775764798.524327:0:64922:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x10:0x0] 00000008:00010000:1.0:1775764798.524332:0:64922:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764798.524334:0:64922:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5ab0, osc lock ffff8b2382cf7a20, flags 0x400000400001002 00010000:00010000:1.0:1775764798.524342:0:64922:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764798.524350:0:64922:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00000020:00200000:1.0:1775764798.524358:0:64922:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764798.524361:0:64922:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764798.524368:0:64922:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00000020:00200000:1.0:1775764798.524375:0:64922:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764798.524381:0:64922:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x10:0x0] updating i_size 16 i_blocks 1 00000020:00010000:1.0:1775764798.524384:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764798.524387:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764798.524390:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5ab0 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764798.524395:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764798.524398:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5ab0 (ffff8b2387cba638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764798.524407:0:64922:0:(vvp_io.c:885:vvp_io_read_start()) count:4194288 iocb pos:16 00000020:00010000:1.0:1775764798.524417:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764798.524420:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764798.524422:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5330 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764798.524426:0:64922:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764798.524435:0:64922:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00000020:00010000:1.0:1775764798.524449:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764798.524618:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5330 (ffff8b2387cba638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764798.524625:0:64922:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:1.0:1775764798.524651:0:64922:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 5 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764798.524656:0:64922:0:(file.c:2206:ll_file_io_generic()) f25.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:1.0:1775764798.524659:0:64922:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:1.0:1775764798.524712:0:64922:0:(file.c:2454:do_file_read_iter()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0], ppos: 16, count: 4194304 00000080:02000000:1.0:1775764798.524719:0:64922:0:(pcc.c:2487:pcc_file_read_iter()) *** cfs_fail_loc=1411, val=0*** 00000080:00200000:1.0:1775764798.524736:0:64922:0:(file.c:2058:ll_file_io_generic()) f25.sanity-pcc: read ppos: 16, bytes: 4194304 00000080:00200000:1.0:1775764798.524739:0:64922:0:(file.c:1815:ll_io_set_mirror()) f25.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764798.524744:0:64922:0:(cl_io.c:214:cl_io_rw_init()) header@00000000ab8aad76[0x110, 2, [0x200003ab4:0x10:0x0] hash] 00000020:00200000:1.0:1775764798.524746:0:64922:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [16, 4194320) 0 0 00000080:00200000:1.0:1775764798.524748:0:64922:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764798.524755:0:64922:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x10:0x0]: flr state: 10, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764798.524758:0:64922:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:1.0:1775764798.524761:0:64922:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 16 chunk: [16, 4194320) 4194320, 4194304 00020000:00200000:1.0:1775764798.524764:0:64922:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764798.524779:0:64922:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 1024] 00000008:00010000:1.0:1775764798.524786:0:64922:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764798.524789:0:64922:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5330, osc lock ffff8b2382cf6e40, flags 0x400000000000000 00010000:00010000:1.0:1775764798.524798:0:64922:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764798.524820:0:64922:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4198399) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764798.524829:0:64922:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764798.524836:0:64922:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764798.524842:0:64922:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00000080:00200000:1.0:1775764798.524848:0:64922:0:(vvp_io.c:826:vvp_io_read_start()) f25.sanity-pcc: read [16, 4194320) 00000001:00010000:1.0:1775764798.524854:0:64922:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x10:0x0] 00000008:00010000:1.0:1775764798.524858:0:64922:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764798.524860:0:64922:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5ab0, osc lock ffff8b2382cf7a20, flags 0x400000400001002 00010000:00010000:1.0:1775764798.524866:0:64922:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764798.524873:0:64922:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00000020:00200000:1.0:1775764798.524880:0:64922:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764798.524883:0:64922:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764798.524891:0:64922:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 6/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00000020:00200000:1.0:1775764798.524897:0:64922:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764798.524901:0:64922:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x10:0x0] updating i_size 16 i_blocks 1 00000020:00010000:1.0:1775764798.524904:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764798.524925:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764798.524928:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5ab0 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764798.524930:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764798.524933:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5ab0 (ffff8b2387cba638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764798.524937:0:64922:0:(vvp_io.c:885:vvp_io_read_start()) count:4194304 iocb pos:16 00000020:00010000:1.0:1775764798.524943:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764798.524945:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764798.524947:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5330 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764798.524951:0:64922:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764798.524959:0:64922:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a8cbb627/0x73cfd86cfc17bf91 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030524ca expref: -99 pid: 64571 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00000020:00010000:1.0:1775764798.524968:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764798.524971:0:64922:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5330 (ffff8b2387cba638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764798.524974:0:64922:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:1.0:1775764798.524986:0:64922:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x10:0x0] ignore/verify layout 0/0, layout version 5 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764798.524989:0:64922:0:(file.c:2206:ll_file_io_generic()) f25.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:1.0:1775764798.524992:0:64922:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:1.0:1775764798.525082:0:64922:0:(file.c:447:ll_file_release()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148), flags 100000 00010000:00010000:1.0:1775764798.525103:0:64922:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17c01d lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030526a6 expref: -99 pid: 64922 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764798.657050:0:64932:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764798.657084:0:64932:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764798.657104:0:64932:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1775764798.657122:0:64932:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:2.0:1775764798.657136:0:64932:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764798.657146:0:64932:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764798.657150:0:64932:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:2.0:1775764798.657158:0:64932:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01fc7bf8 00010000:00010000:2.0:1775764798.657162:0:64932:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1775764798.657170:0:64932:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:2.0:1775764798.657188:0:64932:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764798.657197:0:64932:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d25.sanity-pcc, flags=267 00000080:00200000:2.0:1775764798.657204:0:64932:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988), inode mode 41ed mask 44 00000080:00200008:2.0:1775764798.657211:0:64932:0:(file.c:1057:ll_file_open()) START file d25.sanity-pcc:[0x200003ab4:0xf:0x0](ffff8b2389a61988), flags 304000 00000002:00010000:2.0:1775764798.657227:0:64932:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xf:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240304001 00010000:00010000:2.0:1775764798.657279:0:64932:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c024 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64932 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764798.657287:0:64932:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c024 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64932 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764798.657298:0:64932:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764798.657311:0:64932:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4e00 x1862022098658688/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764798.657321:0:64932:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c024 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64932 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764798.657345:0:64932:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64932:1862022098658688:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764798.658008:0:64932:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:2.0:1775764798.665166:0:64932:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64932:1862022098658688:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764798.665198:0:64932:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c024 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030526bb expref: -99 pid: 64932 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764798.665208:0:64932:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c024 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030526bb expref: -99 pid: 64932 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764798.665220:0:64932:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c024 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030526bb expref: -99 pid: 64932 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764798.665229:0:64932:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c024 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030526bb expref: -99 pid: 64932 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764798.665243:0:64932:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23b9ea4e00 x1862022098658688/t34359739265(34359739265) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764814 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764798.665256:0:64932:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4e00 x1862022098658688/t34359739265(34359739265) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764814 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764798.665266:0:64932:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c024 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030526bb expref: -99 pid: 64932 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764798.665275:0:64932:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/15 (0 0) 00000080:00200000:2.0:1775764798.665282:0:64932:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764798.665306:0:64932:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0xf:0x0], updating i_size 7680 00000080:00010000:2.0:1775764798.665312:0:64932:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xf:0x0](ffff8b2389a61988) for lock 0x73cfd86cfc17c024 00000080:00010000:2.0:1775764798.665320:0:64932:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c024 from it ffffa60b01fc7bd0 00010000:00010000:2.0:1775764798.665324:0:64932:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c024 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030526bb expref: -99 pid: 64932 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764798.665333:0:64932:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c024 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030526bb expref: -99 pid: 64932 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:2.0:1775764798.665363:0:64932:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4e00 x1862022098658688/t34359739265(34359739265) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764814 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764798.665710:0:64932:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988) cmd=901066fc arg=7ffdc89c6d00 00000080:00200008:2.0:1775764798.665750:0:64932:0:(file.c:447:ll_file_release()) START file d25.sanity-pcc:[0x200003ab4:0xf:0x0](ffff8b2389a61988), flags 304000 00010000:00010000:2.0:1775764798.665767:0:64932:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c024 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030526bb expref: -99 pid: 64932 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200008:2.0:1775764798.782743:0:64938:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764798.782758:0:64938:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764798.782780:0:64938:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764798.782801:0:64938:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:2.0:1775764798.782819:0:64938:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:2.0:1775764798.782828:0:64938:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764798.782837:0:64938:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764798.782841:0:64938:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:2.0:1775764798.782847:0:64938:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01f57cb8 00010000:00010000:2.0:1775764798.782852:0:64938:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:2.0:1775764798.782861:0:64938:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:2.0:1775764798.782887:0:64938:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764798.782894:0:64938:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764798.782900:0:64938:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:2.0:1775764798.782926:0:64938:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:2.0:1775764798.782933:0:64938:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764798.782938:0:64938:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764798.782940:0:64938:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:2.0:1775764798.782944:0:64938:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01f57b58 00010000:00010000:2.0:1775764798.782946:0:64938:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:2.0:1775764798.782953:0:64938:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:2.0:1775764798.782960:0:64938:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764798.782967:0:64938:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d25.sanity-pcc, flags=8 00000080:00200008:2.0:1775764798.782974:0:64938:0:(file.c:6226:ll_getattr_dentry()) START file d25.sanity-pcc:[0x200003ab4:0xf:0x0](ffff8b2389a61988), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764798.782977:0:64938:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988),name=d25.sanity-pcc 00000002:00010000:2.0:1775764798.782988:0:64938:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0xf:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764798.782995:0:64938:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bff3 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305260c expref: -99 pid: 64892 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764798.783004:0:64938:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bff3 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305260c expref: -99 pid: 64892 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764798.783011:0:64938:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764798.783015:0:64938:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0xf:0x0](ffff8b2389a61988) 00000080:00010000:2.0:1775764798.783017:0:64938:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0xf:0x0](ffff8b2389a61988) for lock 0x73cfd86cfc17bff3 00000080:00010000:2.0:1775764798.783021:0:64938:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bff3 from it ffffa60b01f57cb8 00010000:00010000:2.0:1775764798.783023:0:64938:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bff3 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305260c expref: -99 pid: 64892 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764798.783031:0:64938:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bff3 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305260c expref: -99 pid: 64892 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764798.783047:0:64938:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764798.783053:0:64938:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764798.783059:0:64938:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:2.0:1775764798.783066:0:64938:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:2.0:1775764798.783073:0:64938:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764798.783076:0:64938:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764798.783078:0:64938:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:2.0:1775764798.783082:0:64938:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01f57b58 00010000:00010000:2.0:1775764798.783084:0:64938:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:2.0:1775764798.783088:0:64938:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:2.0:1775764798.783093:0:64938:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764798.783095:0:64938:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d25.sanity-pcc, flags=24 00000080:00200000:2.0:1775764798.783098:0:64938:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764798.783100:0:64938:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f25.sanity-pcc, flags=8 00000080:00200008:2.0:1775764798.783103:0:64938:0:(file.c:6226:ll_getattr_dentry()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764798.783105:0:64938:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x10:0x0](ffff8b2390ec1148),name=f25.sanity-pcc 00000002:00010000:2.0:1775764798.783112:0:64938:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x10:0x0]) in obj [0x200003ab4:0xf:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764798.783119:0:64938:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c016 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052691 expref: -99 pid: 64909 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764798.783127:0:64938:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c016 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052691 expref: -99 pid: 64909 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764798.783133:0:64938:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200003ab4:0xf:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764798.783136:0:64938:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) 00000080:00010000:2.0:1775764798.783138:0:64938:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x10:0x0](ffff8b2390ec1148) for lock 0x73cfd86cfc17c016 00000080:00010000:2.0:1775764798.783141:0:64938:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c016 from it ffffa60b01f57cb8 00010000:00010000:2.0:1775764798.783144:0:64938:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c016 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052691 expref: -99 pid: 64909 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764798.783150:0:64938:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c016 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052691 expref: -99 pid: 64909 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764798.783179:0:64938:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764798.783185:0:64938:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764798.783191:0:64938:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:2.0:1775764798.783198:0:64938:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:2.0:1775764798.783202:0:64938:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764798.783205:0:64938:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764798.783206:0:64938:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:2.0:1775764798.783209:0:64938:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01f57bf8 00010000:00010000:2.0:1775764798.783211:0:64938:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:2.0:1775764798.783215:0:64938:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:2.0:1775764798.783219:0:64938:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764798.783222:0:64938:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d25.sanity-pcc, flags=25 00000080:00200000:2.0:1775764798.783224:0:64938:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764798.783226:0:64938:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f25.sanity-pcc, flags=265 00000080:00200000:2.0:1775764798.783227:0:64938:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x10:0x0](ffff8b2390ec1148), inode mode 81a4 mask 46 00000080:00200008:2.0:1775764798.783231:0:64938:0:(file.c:1057:ll_file_open()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148), flags 154402 00000002:00010000:2.0:1775764798.783241:0:64938:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x10:0x0]) in obj [0x200003ab4:0xf:0x0], intent: open flags 0104240114403 00010000:00010000:2.0:1775764798.783281:0:64938:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c02b lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 64938 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764798.783286:0:64938:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c02b lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x0 pid: 64938 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764798.783295:0:64938:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764798.783304:0:64938:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5180 x1862022098658944/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764798.783311:0:64938:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c02b lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x0 pid: 64938 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764798.783330:0:64938:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64938:1862022098658944:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764798.783770:0:64938:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238394f180 going to sleep for 11 seconds 00000100:00100000:1.0:1775764798.787088:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764798.787713:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764798.787719:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c016 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03052691 expref: -99 pid: 64909 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764798.787728:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2381496a00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764798.787732:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c016 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052691 expref: -99 pid: 64909 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764798.787741:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c016 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052691 expref: -99 pid: 64909 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764798.787750:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c016 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03052691 expref: -99 pid: 64909 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764798.787776:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17c01d lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030526a6 expref: -99 pid: 64922 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764798.787793:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c008 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052652 expref: -99 pid: 64892 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764798.787840:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c016 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052691 expref: -99 pid: 64909 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764798.787845:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764798.787871:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c016 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052691 expref: -99 pid: 64909 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764798.787876:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c016 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x10:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052691 expref: -99 pid: 64909 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764798.787890:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764798.787891:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764798.787892:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764798.787898:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764798.787925:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098659072:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764798.791320:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098659072:192.168.202.128@tcp:103:ldlm_bl.0 00000080:00200080:1.0:1775764798.792255:0:64938:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x10:0x0](ffff8b2390ec1148) cmd=c00466fc arg=7ffea4dacbbc 00000080:00200008:1.0:1775764798.792322:0:64938:0:(file.c:447:ll_file_release()) START file f25.sanity-pcc:[0x200003ab4:0x10:0x0](ffff8b2390ec1148), flags 154002 00010000:00010000:1.0:1775764798.792340:0:64938:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c02b lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x10:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030526d0 expref: -99 pid: 64938 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764798.879337:0:64948:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764798.879366:0:64948:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764798.879385:0:64948:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:1.0:1775764798.879401:0:64948:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:1.0:1775764798.879410:0:64948:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764798.879417:0:64948:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764798.879421:0:64948:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:1.0:1775764798.879428:0:64948:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01fefbf8 00010000:00010000:1.0:1775764798.879432:0:64948:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:1.0:1775764798.879441:0:64948:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:1.0:1775764798.879734:0:64948:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764798.879743:0:64948:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d25.sanity-pcc, flags=267 00000080:00200000:1.0:1775764798.879749:0:64948:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988), inode mode 41ed mask 44 00000080:00200008:1.0:1775764798.879755:0:64948:0:(file.c:1057:ll_file_open()) START file d25.sanity-pcc:[0x200003ab4:0xf:0x0](ffff8b2389a61988), flags 304000 00000080:00200080:1.0:1775764798.879787:0:64948:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200003ab4:0xf:0x0](ffff8b2389a61988) cmd=901066fc arg=7ffda834d9f0 00000100:00100000:1.0:1775764798.879855:0:64948:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:64948:1862022098659200:192.168.202.128@tcp:34:lfs.0 00000100:00100000:1.0:1775764798.880159:0:64948:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654180 going to sleep for 11 seconds 00010000:00010000:2.0:1775764800.439650:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764803.513501:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764804.542770:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764805.559557:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764806.583817:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764807.607654:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764808.633056:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764809.657491:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764810.684282:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1775764811.539280:0:65649:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1775764811.539294:0:65649:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764811.539319:0:65649:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764811.539339:0:65649:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00010000:00010000:2.0:1775764811.539355:0:65649:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00000080:00200000:2.0:1775764811.539363:0:65649:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764811.539372:0:65649:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764811.539375:0:65649:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:2.0:1775764811.539381:0:65649:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01e03cb8 00010000:00010000:2.0:1775764811.539385:0:65649:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00010000:00010000:2.0:1775764811.539395:0:65649:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00000080:00200008:2.0:1775764811.540449:0:65649:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1775764811.551112:0:65650:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764811.551298:0:65650:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764811.551321:0:65650:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:2.0:1775764811.551337:0:65650:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00000080:00200000:2.0:1775764811.551345:0:65650:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764811.551352:0:65650:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764811.551356:0:65650:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:2.0:1775764811.551363:0:65650:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01dabbb0 00010000:00010000:2.0:1775764811.551367:0:65650:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:2.0:1775764811.551375:0:65650:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:2.0:1775764811.551391:0:65650:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764811.551401:0:65650:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 100000 00000080:00200000:2.0:1775764811.551429:0:65650:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764811.551436:0:65650:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764811.551442:0:65650:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:2.0:1775764811.551552:0:65650:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:2.0:1775764811.551648:0:65650:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764811.551655:0:65650:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764811.551657:0:65650:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:2.0:1775764811.551662:0:65650:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01dabbf8 00010000:00010000:2.0:1775764811.551665:0:65650:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:2.0:1775764811.551672:0:65650:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:2.0:1775764811.551681:0:65650:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764811.551688:0:65650:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=.lustre, flags=17 00000080:00200000:2.0:1775764811.551694:0:65650:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000002:0x1:0x0](ffff8b2389909988), inode mode 41ed mask 1 00000080:00200000:2.0:1775764811.551706:0:65650:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=fid, dir=[0x200000002:0x1:0x0](ffff8b2389909988), file ffff8b2388730600, open_flags 8000, mode 0 opened 0 00000080:00200000:2.0:1775764811.551712:0:65650:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=fid, dir=[0x200000002:0x1:0x0](ffff8b2389909988), intent=open 00000002:00010000:2.0:1775764811.551725:0:65650:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: fid,[0x0:0x0:0x0]) in obj [0x200000002:0x1:0x0], intent: open flags 0100001 00010000:00010000:2.0:1775764811.551880:0:65650:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c039 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65650 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.551888:0:65650:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c039 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65650 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764811.551900:0:65650:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764811.551911:0:65650:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6680 x1862022098661248/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764811.551921:0:65650:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c039 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65650 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764811.551943:0:65650:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:65650:1862022098661248:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764811.552349:0:65650:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2391043240 going to sleep for 11 seconds 00000080:00200000:1.0:1775764811.638345:0:65658:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764811.638374:0:65658:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764811.638393:0:65658:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1775764811.638409:0:65658:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:1.0:1775764811.638416:0:65658:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764811.638425:0:65658:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764811.638429:0:65658:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17bf7c 00000080:00010000:1.0:1775764811.638436:0:65658:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17bf7c from it ffffa60b01de3bb0 00010000:00010000:1.0:1775764811.638440:0:65658:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1775764811.638596:0:65658:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea400/0x73cfd86cfc17bf7c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052484 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000080:00200000:1.0:1775764811.638612:0:65658:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764811.638618:0:65658:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200080:1.0:1775764811.638645:0:65658:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=8080667f arg=7fffcfa2ba50 00000080:00200008:1.0:1775764811.638660:0:65658:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000100:00100000:1.0:1775764811.657590:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:2.0:1775764811.745047:0:65664:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad180 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:65664:1862022098662400:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764811.745074:0:65664:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.745086:0:65664:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.745098:0:65664:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.745114:0:65664:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.745277:0:65664:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.745286:0:65664:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.745294:0:65664:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764811.745310:0:65664:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b2382dad180 x1862022098662400/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764827 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764811.745323:0:65664:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.745337:0:65664:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764811.745343:0:65664:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764811.745372:0:65664:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764811.745377:0:65664:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c04e 00000080:00010000:2.0:1775764811.745382:0:65664:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764811.745385:0:65664:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c04e from it ffffa60b01e6bcb8 00010000:00010000:2.0:1775764811.745388:0:65664:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.745403:0:65664:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764811.745784:0:65664:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764811.748072:0:65664:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764811.748100:0:65664:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764811.748283:0:65664:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764811.748294:0:65664:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764811.748307:0:65664:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764811.748311:0:65664:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c04e 00000080:00010000:2.0:1775764811.748317:0:65664:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c04e from it ffffa60b01e6bb58 00010000:00010000:2.0:1775764811.748321:0:65664:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764811.748329:0:65664:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764811.748344:0:65664:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764811.748355:0:65664:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=8 00000080:00200008:2.0:1775764811.748369:0:65664:0:(file.c:6226:ll_getattr_dentry()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764811.748375:0:65664:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88),name=f26.sanity-pcc 00000002:00010000:2.0:1775764811.748392:0:65664:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764811.748401:0:65664:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c040 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052739 expref: -99 pid: 65417 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764811.748411:0:65664:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c040 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052739 expref: -99 pid: 65417 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764811.748418:0:65664:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764811.748422:0:65664:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00010000:2.0:1775764811.748424:0:65664:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c040 00000080:00010000:2.0:1775764811.748427:0:65664:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c040 from it ffffa60b01e6bcb8 00010000:00010000:2.0:1775764811.748430:0:65664:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c040 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052739 expref: -99 pid: 65417 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764811.748436:0:65664:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c040 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052739 expref: -99 pid: 65417 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764811.749313:0:65664:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1775764811.749326:0:65664:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x12:0x0] 00000008:00010000:2.0:1775764811.749353:0:65664:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764811.749357:0:65664:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8bb0, osc lock ffff8b238a3f8000, flags 0x400001002 00010000:00010000:2.0:1775764811.749947:0:65664:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764811.749967:0:65664:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1775764811.749980:0:65664:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764811.749983:0:65664:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764811.749991:0:65664:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 5/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1775764811.750004:0:65664:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764811.750015:0:65664:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x12:0x0] updating i_size 16 i_blocks 1 00000020:00010000:2.0:1775764811.750020:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764811.750023:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764811.750026:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8bb0 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764811.750030:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764811.750032:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8bb0 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764811.750059:0:65664:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764811.750122:0:65664:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764811.750637:0:65664:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764811.750650:0:65664:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764811.750662:0:65664:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764811.750668:0:65664:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764811.750676:0:65664:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764811.750852:0:65664:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c04e 00000080:00010000:2.0:1775764811.750860:0:65664:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c04e from it ffffa60b01e6bbf8 00010000:00010000:2.0:1775764811.750864:0:65664:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764811.750877:0:65664:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c04e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052755 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764811.750890:0:65664:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764811.750899:0:65664:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=265 00000080:00200000:2.0:1775764811.750906:0:65664:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88), inode mode 81a4 mask 44 00000080:00200008:2.0:1775764811.750913:0:65664:0:(file.c:1057:ll_file_open()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100000 00000002:00010000:2.0:1775764811.750929:0:65664:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 04240100001 00010000:00010000:2.0:1775764811.750979:0:65664:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c055 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65664 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.750986:0:65664:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c055 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65664 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764811.750998:0:65664:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764811.751009:0:65664:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6d80 x1862022098662656/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764811.751019:0:65664:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c055 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65664 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764811.751043:0:65664:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:65664:1862022098662656:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764811.752379:0:65664:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0faacc0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764811.788448:0:65664:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:65664:1862022098662656:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764811.788521:0:65664:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c055 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052763 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.788538:0:65664:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c055 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052763 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.788551:0:65664:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c055 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052763 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.788562:0:65664:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c055 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052763 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764811.788578:0:65664:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23b9ea6d80 x1862022098662656/t34359739271(34359739271) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764827 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764811.788590:0:65664:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea6d80 x1862022098662656/t34359739271(34359739271) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764827 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764811.788601:0:65664:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c055 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052763 expref: -99 pid: 65664 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764811.788614:0:65664:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c055 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052763 expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.788624:0:65664:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/18 (0 0) 00000080:00200000:2.0:1775764811.788630:0:65664:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764811.788739:0:65664:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x12:0x0]: layout version change: 0 -> 0 00000080:00010000:2.0:1775764811.788750:0:65664:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c055 00000080:00010000:2.0:1775764811.788758:0:65664:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c055 from it ffffa60b01e6bbf0 00010000:00010000:2.0:1775764811.788762:0:65664:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c055 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052763 expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.788773:0:65664:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c055 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052763 expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00100000:2.0:1775764811.788810:0:65664:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea6d80 x1862022098662656/t34359739271(34359739271) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764827 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764811.788928:0:65664:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88) cmd=400866fc arg=7ffe470c1048 00000080:00010000:2.0:1775764811.788963:0:65664:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000002:00010000:2.0:1775764811.788973:0:65664:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200003ab4:0x12:0x0], intent: layout flags 02 00010000:00010000:2.0:1775764811.788988:0:65664:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c055 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052763 expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764811.789010:0:65664:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x12:0x0]: losing layout lock 00010000:00010000:2.0:1775764811.789028:0:65664:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c040 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052739 expref: -99 pid: 65417 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764811.789315:0:65664:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x12:0x0]: losing layout lock 00010000:00010000:2.0:1775764811.789355:0:65664:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c055 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052763 expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764811.789363:0:65664:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c040 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052739 expref: -99 pid: 65417 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764811.789369:0:65664:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1775764811.789372:0:65664:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c055 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052763 expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764811.789387:0:65664:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c040 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052739 expref: -99 pid: 65417 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764811.789418:0:65664:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x12:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65664 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.789425:0:65664:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65664 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764811.789436:0:65664:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764811.789444:0:65664:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6300 x1862022098662912/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 384/880 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764811.789573:0:65664:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65664 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764811.789599:0:65664:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:65664:1862022098662912:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764811.789758:0:65664:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0faacc0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764811.828725:0:65664:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:65664:1862022098662912:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764811.828767:0:65664:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 4/0,1 mode: --/EX res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.828780:0:65664:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 4/0,1 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.828800:0:65664:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.828812:0:65664:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.828830:0:65664:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.828837:0:65664:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.828844:0:65664:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764811.828856:0:65664:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea6300 x1862022098662912/t34359739272(34359739272) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 384/600 e 0 to 0 dl 1775764827 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764811.828874:0:65664:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764811.828887:0:65664:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.828899:0:65664:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/18 (0 0) 00000080:00200000:2.0:1775764811.828905:0:65664:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x12:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764811.828915:0:65664:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c05c 00000080:00010000:2.0:1775764811.828923:0:65664:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c05c from it ffffa60b01e6bc10 00010000:00010000:2.0:1775764811.828925:0:65664:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764811.828933:0:65664:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764811.828958:0:65664:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764811.828969:0:65664:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764811.828977:0:65664:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x12:0x0](00000000580307f2) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764811.829283:0:65664:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764811.829292:0:65664:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764811.829321:0:65664:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764811.829332:0:65664:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:2.0:1775764811.829368:0:65664:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 000000005c374b27 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:2.0:1775764811.829383:0:65664:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 000000005c374b27 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764811.829429:0:65664:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:2.0:1775764811.829551:0:65664:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea5880 - object 0x240000400:284 offset 0<>16 00000100:00100000:1.0:1775764811.845366:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098663296:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764811.845378:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098663168:192.168.202.128@tcp:400:kworker.0 00000080:00200000:2.0:1775764811.845861:0:65664:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764811.845949:0:65664:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764811.845962:0:65664:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764811.845972:0:65664:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x12:0x0](00000000580307f2) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764811.846037:0:65664:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764811.846047:0:65664:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764811.846081:0:65664:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:2.0:1775764811.846099:0:65664:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 16 blocks 1 [cma]time 1775764811 1775764811 0 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764811.846167:0:65664:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x12:0x0]: layout version change: 4294967294 -> 2 00010000:00010000:2.0:1775764811.846178:0:65664:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764811.846187:0:65664:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:2.0:1775764811.847283:0:65664:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 0, count: 1048576 00000080:00200000:2.0:1775764811.847309:0:65664:0:(rw.c:1928:ll_readpage()) fast read pgno: 0 00000080:00200000:2.0:1775764811.847338:0:65664:0:(file.c:2058:ll_file_io_generic()) f26.sanity-pcc: read ppos: 0, bytes: 1048576 00000080:00200000:2.0:1775764811.847343:0:65664:0:(file.c:1815:ll_io_set_mirror()) f26.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764811.850011:0:65664:0:(cl_io.c:214:cl_io_rw_init()) header@000000002eb11a92[0x110, 1, [0x200003ab4:0x12:0x0] hash] 00000020:00200000:2.0:1775764811.850016:0:65664:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 1048576) 0 0 00000080:00200000:2.0:1775764811.850024:0:65664:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764811.850041:0:65664:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x12:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764811.850045:0:65664:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764811.850055:0:65664:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:2.0:1775764811.850059:0:65664:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764811.850107:0:65664:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:2.0:1775764811.850123:0:65664:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764811.850129:0:65664:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8b30, osc lock ffff8b238a3f8000, flags 0x400000000000000 00010000:00010000:2.0:1775764811.850905:0:65664:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764811.850924:0:65664:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764811.850938:0:65664:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:2.0:1775764811.850951:0:65664:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 16, setting rss=16; kms=16, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764811.850962:0:65664:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764811.850971:0:65664:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764811.850981:0:65664:0:(vvp_io.c:826:vvp_io_read_start()) f26.sanity-pcc: read [0, 1048576) 00000001:00010000:2.0:1775764811.851002:0:65664:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x12:0x0] 00000008:00010000:2.0:1775764811.851010:0:65664:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764811.851013:0:65664:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8cb0, osc lock ffff8b238a3f8be0, flags 0x400000400001002 00010000:00010000:2.0:1775764811.851022:0:65664:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764811.851031:0:65664:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:2.0:1775764811.851041:0:65664:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764811.851044:0:65664:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764811.851051:0:65664:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:2.0:1775764811.851058:0:65664:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764811.851065:0:65664:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x12:0x0] updating i_size 16 i_blocks 1 00000020:00010000:2.0:1775764811.851068:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764811.851072:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764811.851075:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8cb0 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764811.851079:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764811.851082:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8cb0 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764811.851093:0:65664:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:0 00000008:00010000:2.0:1775764811.851157:0:65664:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b2383c9e180 00000008:00010000:2.0:1775764811.851210:0:46654:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b2383c9e180 00010000:00010000:2.0:1775764811.851232:0:46654:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:2.0:1775764811.851282:0:46654:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea4700 - object 0x240000400:284 offset 0<>4096 00000020:00010000:2.0:1775764811.860943:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764811.860954:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764811.860959:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8b30 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764811.860970:0:65664:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764811.860989:0:65664:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:2.0:1775764811.861008:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764811.861011:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8b30 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764811.861023:0:65664:0:(vvp_io.c:517:vvp_io_advance()) advancing 16 bytes 00000080:00200000:2.0:1775764811.861063:0:65664:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764811.861069:0:65664:0:(file.c:2206:ll_file_io_generic()) f26.sanity-pcc: 1 io complete with rc: 0, result: 16, restart: 0 00000080:00200000:2.0:1775764811.861074:0:65664:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 16 00000080:00200008:2.0:1775764811.862345:0:65664:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 16, count: 1048576 00000080:00200000:2.0:1775764811.862369:0:65664:0:(file.c:2058:ll_file_io_generic()) f26.sanity-pcc: read ppos: 16, bytes: 1048576 00000080:00200000:2.0:1775764811.862373:0:65664:0:(file.c:1815:ll_io_set_mirror()) f26.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764811.862380:0:65664:0:(cl_io.c:214:cl_io_rw_init()) header@000000002eb11a92[0x110, 2, [0x200003ab4:0x12:0x0] hash] 00000020:00200000:2.0:1775764811.862382:0:65664:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [16, 1048592) 0 0 00000080:00200000:2.0:1775764811.862385:0:65664:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764811.862399:0:65664:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x12:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764811.862402:0:65664:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764811.862408:0:65664:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 16 chunk: [16, 1048592) 1048592, 1048576 00020000:00200000:2.0:1775764811.862412:0:65664:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764811.862436:0:65664:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 256] 00000008:00010000:2.0:1775764811.862449:0:65664:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764811.862466:0:65664:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8b30, osc lock ffff8b238a3f8000, flags 0x400000000000000 00010000:00010000:2.0:1775764811.862499:0:65664:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764811.862514:0:65664:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1052671) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764811.862526:0:65664:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764811.862535:0:65664:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764811.862542:0:65664:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764811.862550:0:65664:0:(vvp_io.c:826:vvp_io_read_start()) f26.sanity-pcc: read [16, 1048592) 00000001:00010000:2.0:1775764811.862564:0:65664:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x12:0x0] 00000008:00010000:2.0:1775764811.862569:0:65664:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764811.862571:0:65664:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8cb0, osc lock ffff8b238a3f8be0, flags 0x400000400001002 00010000:00010000:2.0:1775764811.862576:0:65664:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764811.862584:0:65664:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:2.0:1775764811.862591:0:65664:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764811.862595:0:65664:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764811.862603:0:65664:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:2.0:1775764811.862609:0:65664:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764811.862614:0:65664:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x12:0x0] updating i_size 16 i_blocks 1 00000020:00010000:2.0:1775764811.862617:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764811.862619:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764811.862622:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8cb0 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764811.862624:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764811.862627:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8cb0 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764811.862631:0:65664:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:16 00000020:00010000:2.0:1775764811.862638:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764811.862640:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764811.862642:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8b30 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764811.862646:0:65664:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764811.862654:0:65664:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00010000:2.0:1775764811.862666:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764811.862668:0:65664:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8b30 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764811.862672:0:65664:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1775764811.862696:0:65664:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764811.862699:0:65664:0:(file.c:2206:ll_file_io_generic()) f26.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1775764811.862701:0:65664:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:2.0:1775764811.862891:0:65664:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100000 00000002:00100000:2.0:1775764811.862948:0:65664:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea6d80 x1862022098662656/t34359739271(34359739271) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764827 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764811.862961:0:65664:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764811.862966:0:65664:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5c00 x1862022098663680/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764811.862988:0:65664:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:65664:1862022098663680:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764811.863071:0:65664:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0faacc0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764811.873088:0:65664:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:65664:1862022098663680:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764811.873126:0:65664:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea6d80 x1862022098662656/t34359739271(34359739271) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764827 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764811.999005:0:65674:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:65674:1862022098663808:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764811.999094:0:65674:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000100:00100000:2.0:1775764812.600874:0:65690:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:65690:1862022098664064:192.168.202.128@tcp:101:multiop.0 00010000:00010000:2.0:1775764812.600915:0:65690:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.600941:0:65690:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.600949:0:65690:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.600962:0:65690:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.600974:0:65690:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.600987:0:65690:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.600995:0:65690:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.601001:0:65690:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764812.601013:0:65690:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=2b, status=0 req@ffff8b2382dadf80 x1862022098664064/t34359739274(34359739274) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/864 e 0 to 0 dl 1775764828 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764812.601026:0:65690:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382dadf80 x1862022098664064/t34359739274(34359739274) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/864 e 0 to 0 dl 1775764828 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764812.601040:0:65690:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764812.601054:0:65690:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.601064:0:65690:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/18 (0 0) 00000080:00200000:2.0:1775764812.601070:0:65690:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764812.601196:0:65690:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0x12:0x0](ffff8b238990db88): rc = 0 00000080:00200000:2.0:1775764812.601217:0:65690:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x12:0x0]: layout version change: 2 -> 2 00000080:00010000:2.0:1775764812.601226:0:65690:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c063 00000080:00010000:2.0:1775764812.601240:0:65690:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00200008:2.0:1775764812.601257:0:65690:0:(file.c:1057:ll_file_open()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100102 00000002:00100000:2.0:1775764812.601276:0:65690:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382dadf80 x1862022098664064/t34359739274(34359739274) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764828 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764812.601313:0:65690:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c063 from it ffff8b23891e8f60 00010000:00010000:2.0:1775764812.601319:0:65690:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764812.601327:0:65690:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764812.601357:0:65690:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88), inode mode 81a4 mask 46 00000100:00100000:2.0:1775764815.319193:0:65717:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 34359739274 gen 1 00000100:00100000:2.0:1775764815.319216:0:65717:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23b9ea6680 x1862022098661248/t34359739268(34359739268) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/4688 e 0 to 0 dl 1775764827 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764815.319252:0:65717:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b2382dadf80 x1862022098664064/t34359739274(34359739274) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764828 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764815.319265:0:65717:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:65717:1862022098664192:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764815.319287:0:65717:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764815.319298:0:65717:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764815.319311:0:65717:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764815.319323:0:65717:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764815.319335:0:65717:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764815.319343:0:65717:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764815.319350:0:65717:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764815.319359:0:65717:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b2382daf480 x1862022098664192/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764831 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764815.319382:0:65717:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764815.319395:0:65717:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764815.319402:0:65717:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764815.319432:0:65717:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764815.319439:0:65717:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c06a 00000080:00010000:2.0:1775764815.319445:0:65717:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764815.319448:0:65717:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c06a from it ffffa60b01ebbcb8 00010000:00010000:2.0:1775764815.319579:0:65717:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764815.319593:0:65717:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1775764815.423601:0:65723:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764815.423614:0:65723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764815.423635:0:65723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764815.423652:0:65723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764815.423667:0:65723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764815.423676:0:65723:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764815.423697:0:65723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764815.423701:0:65723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c06a 00000080:00010000:2.0:1775764815.423708:0:65723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c06a from it ffffa60b01eebcb8 00010000:00010000:2.0:1775764815.423712:0:65723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764815.423721:0:65723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764815.423744:0:65723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764815.423749:0:65723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764815.423754:0:65723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764815.423794:0:65723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764815.423801:0:65723:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764815.423804:0:65723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764815.423807:0:65723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c06a 00000080:00010000:2.0:1775764815.423810:0:65723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c06a from it ffffa60b01eebb58 00010000:00010000:2.0:1775764815.423813:0:65723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764815.423820:0:65723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764815.423827:0:65723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764815.423833:0:65723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=8 00000080:00200008:2.0:1775764815.423839:0:65723:0:(file.c:6226:ll_getattr_dentry()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764815.423842:0:65723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88),name=f26.sanity-pcc 00000002:00010000:2.0:1775764815.423852:0:65723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764815.423859:0:65723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764815.423867:0:65723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764815.423873:0:65723:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764815.423876:0:65723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00010000:2.0:1775764815.423878:0:65723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c063 00000080:00010000:2.0:1775764815.423881:0:65723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c063 from it ffffa60b01eebcb8 00010000:00010000:2.0:1775764815.423884:0:65723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764815.423891:0:65723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764815.423934:0:65723:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:2.0:1775764815.423944:0:65723:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x12:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764815.423947:0:65723:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1775764815.423951:0:65723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x12:0x0] 00000008:00010000:2.0:1775764815.423977:0:65723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764815.423979:0:65723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8b30, osc lock ffff8b238a3f8000, flags 0x400000400001002 00010000:00010000:2.0:1775764815.423995:0:65723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764815.424006:0:65723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:2.0:1775764815.424018:0:65723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764815.424022:0:65723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764815.424030:0:65723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00200000:2.0:1775764815.424040:0:65723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764815.424052:0:65723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x12:0x0] updating i_size 16 i_blocks 1 00000020:00010000:2.0:1775764815.424056:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764815.424060:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764815.424063:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8b30 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764815.424067:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764815.424069:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8b30 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764815.424099:0:65723:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/1, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764815.424124:0:65723:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764815.424131:0:65723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764815.424136:0:65723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764815.424145:0:65723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764815.424150:0:65723:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764815.424154:0:65723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764815.424156:0:65723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c06a 00000080:00010000:2.0:1775764815.424159:0:65723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c06a from it ffffa60b01eebbf8 00010000:00010000:2.0:1775764815.424162:0:65723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764815.424168:0:65723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764815.424176:0:65723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764815.424180:0:65723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=265 00000080:00200000:2.0:1775764815.424183:0:65723:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88), inode mode 81a4 mask 44 00000080:00200008:2.0:1775764815.424189:0:65723:0:(file.c:1057:ll_file_open()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100000 00000002:00010000:2.0:1775764815.424202:0:65723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 04240100001 00010000:00010000:2.0:1775764815.424252:0:65723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65723 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764815.424259:0:65723:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65723 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764815.424270:0:65723:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764815.424282:0:65723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6300 x1862022098664448/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764815.424293:0:65723:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65723 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764815.424316:0:65723:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:65723:1862022098664448:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764815.424517:0:65723:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0faa840 going to sleep for 11 seconds 00000100:00100000:2.0:1775764815.431119:0:65723:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:65723:1862022098664448:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764815.431154:0:65723:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764815.431165:0:65723:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764815.431176:0:65723:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764815.431187:0:65723:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764815.431206:0:65723:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23b9ea6300 x1862022098664448/t34359739275(34359739275) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764831 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764815.431218:0:65723:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea6300 x1862022098664448/t34359739275(34359739275) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764831 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764815.431231:0:65723:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764815.431243:0:65723:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764815.431253:0:65723:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/18 (0 0) 00000080:00200000:2.0:1775764815.431261:0:65723:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764815.431361:0:65723:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x12:0x0]: layout version change: 2 -> 2 00000080:00010000:2.0:1775764815.431372:0:65723:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c071 00000080:00010000:2.0:1775764815.431378:0:65723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c071 from it ffffa60b01eebbf0 00010000:00010000:2.0:1775764815.431382:0:65723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764815.431392:0:65723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00100000:2.0:1775764815.431415:0:65723:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea6300 x1862022098664448/t34359739275(34359739275) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764831 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764815.431524:0:65723:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88) cmd=400866fc arg=7ffc44bcce28 00000080:00200008:2.0:1775764815.432029:0:65723:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 0, count: 1048576 00000080:00200000:2.0:1775764815.432061:0:65723:0:(file.c:2058:ll_file_io_generic()) f26.sanity-pcc: read ppos: 16, bytes: 1048560 00000080:00200000:2.0:1775764815.432066:0:65723:0:(file.c:1815:ll_io_set_mirror()) f26.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764815.432074:0:65723:0:(cl_io.c:214:cl_io_rw_init()) header@000000002eb11a92[0x110, 2, [0x200003ab4:0x12:0x0] hash] 00000020:00200000:2.0:1775764815.432076:0:65723:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [16, 1048576) 0 0 00000080:00200000:2.0:1775764815.432081:0:65723:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764815.432095:0:65723:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x12:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764815.432098:0:65723:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764815.432106:0:65723:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 16 chunk: [16, 1048576) 1048576, 1048560 00020000:00200000:2.0:1775764815.432110:0:65723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764815.432138:0:65723:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:2.0:1775764815.432153:0:65723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764815.432157:0:65723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8b30, osc lock ffff8b238a3f8000, flags 0x400000000000000 00010000:00010000:2.0:1775764815.432176:0:65723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764815.432188:0:65723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764815.432200:0:65723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764815.432208:0:65723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764815.432216:0:65723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764815.432225:0:65723:0:(vvp_io.c:826:vvp_io_read_start()) f26.sanity-pcc: read [16, 1048576) 00000001:00010000:2.0:1775764815.432239:0:65723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x12:0x0] 00000008:00010000:2.0:1775764815.432245:0:65723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764815.432247:0:65723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8cb0, osc lock ffff8b238a3f8be0, flags 0x400000400001002 00010000:00010000:2.0:1775764815.432254:0:65723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764815.432263:0:65723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00200000:2.0:1775764815.432271:0:65723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764815.432274:0:65723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764815.432281:0:65723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00200000:2.0:1775764815.432287:0:65723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764815.432294:0:65723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x12:0x0] updating i_size 16 i_blocks 1 00000020:00010000:2.0:1775764815.432298:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764815.432301:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764815.432304:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8cb0 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764815.432308:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764815.432311:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8cb0 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764815.432319:0:65723:0:(vvp_io.c:885:vvp_io_read_start()) count:1048560 iocb pos:16 00000020:00010000:2.0:1775764815.432329:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764815.432332:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764815.432334:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8b30 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764815.432338:0:65723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764815.432351:0:65723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00010000:2.0:1775764815.432362:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764815.432364:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8b30 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764815.432370:0:65723:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1775764815.432391:0:65723:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764815.432395:0:65723:0:(file.c:2206:ll_file_io_generic()) f26.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1775764815.432398:0:65723:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:2.0:1775764815.432432:0:65723:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 16, count: 1048576 00000080:00200000:2.0:1775764815.432446:0:65723:0:(file.c:2058:ll_file_io_generic()) f26.sanity-pcc: read ppos: 16, bytes: 1048576 00000080:00200000:2.0:1775764815.432449:0:65723:0:(file.c:1815:ll_io_set_mirror()) f26.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764815.432485:0:65723:0:(cl_io.c:214:cl_io_rw_init()) header@000000002eb11a92[0x110, 2, [0x200003ab4:0x12:0x0] hash] 00000020:00200000:2.0:1775764815.432487:0:65723:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [16, 1048592) 0 0 00000080:00200000:2.0:1775764815.432490:0:65723:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764815.432496:0:65723:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x12:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764815.432498:0:65723:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764815.432501:0:65723:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 16 chunk: [16, 1048592) 1048592, 1048576 00020000:00200000:2.0:1775764815.432504:0:65723:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764815.432518:0:65723:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 256] 00000008:00010000:2.0:1775764815.432522:0:65723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764815.432524:0:65723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8b30, osc lock ffff8b238a3f8000, flags 0x400000000000000 00010000:00010000:2.0:1775764815.432532:0:65723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764815.432541:0:65723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1052671) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764815.432553:0:65723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764815.432560:0:65723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764815.432566:0:65723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000080:00200000:2.0:1775764815.432573:0:65723:0:(vvp_io.c:826:vvp_io_read_start()) f26.sanity-pcc: read [16, 1048592) 00000001:00010000:2.0:1775764815.432581:0:65723:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x12:0x0] 00000008:00010000:2.0:1775764815.432585:0:65723:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764815.432587:0:65723:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8cb0, osc lock ffff8b238a3f8be0, flags 0x400000400001002 00010000:00010000:2.0:1775764815.432593:0:65723:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764815.432602:0:65723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00200000:2.0:1775764815.432609:0:65723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764815.432612:0:65723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764815.432618:0:65723:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00200000:2.0:1775764815.432629:0:65723:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764815.432632:0:65723:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x12:0x0] updating i_size 16 i_blocks 1 00000020:00010000:2.0:1775764815.432634:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764815.432636:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764815.432639:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8cb0 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764815.432642:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764815.432643:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8cb0 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764815.432648:0:65723:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:16 00000020:00010000:2.0:1775764815.432653:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764815.432655:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764815.432657:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8b30 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764815.432661:0:65723:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1775764815.432669:0:65723:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000020:00010000:2.0:1775764815.432676:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764815.432678:0:65723:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8b30 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764815.433882:0:65723:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1775764815.433915:0:65723:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764815.433920:0:65723:0:(file.c:2206:ll_file_io_generic()) f26.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1775764815.433924:0:65723:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:2.0:1775764815.434138:0:65723:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100000 00010000:00010000:2.0:1775764815.434163:0:65723:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764815.544369:0:65734:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764815.544397:0:65734:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764815.544415:0:65734:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764815.544431:0:65734:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764815.544439:0:65734:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764815.544447:0:65734:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764815.544450:0:65734:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c06a 00000080:00010000:2.0:1775764815.544528:0:65734:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c06a from it ffffa60b01f8fbb0 00010000:00010000:2.0:1775764815.544532:0:65734:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764815.544541:0:65734:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764815.544557:0:65734:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764815.544564:0:65734:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764815.544591:0:65734:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffea090e500 00000080:00200008:2.0:1775764815.544617:0:65734:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:1.0:1775764815.646183:0:65741:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764815.646214:0:65741:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764815.646233:0:65741:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764815.646250:0:65741:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764815.646258:0:65741:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764815.646267:0:65741:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764815.646270:0:65741:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c06a 00000080:00010000:1.0:1775764815.646278:0:65741:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c06a from it ffffa60b01ecbbf8 00010000:00010000:1.0:1775764815.646282:0:65741:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764815.646296:0:65741:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764815.646314:0:65741:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764815.646323:0:65741:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=265 00000080:00200000:1.0:1775764815.646328:0:65741:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88), inode mode 81a4 mask 44 00000080:00200008:1.0:1775764815.646333:0:65741:0:(file.c:1057:ll_file_open()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100000 00000080:00200008:1.0:1775764815.646360:0:65741:0:(file.c:6226:ll_getattr_dentry()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764815.646365:0:65741:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88),name=f26.sanity-pcc 00000002:00010000:1.0:1775764815.646377:0:65741:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764815.646385:0:65741:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764815.646394:0:65741:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764815.646401:0:65741:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764815.646404:0:65741:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00010000:1.0:1775764815.646407:0:65741:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c063 00000080:00010000:1.0:1775764815.646410:0:65741:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c063 from it ffffa60b01ecbcd8 00010000:00010000:1.0:1775764815.646413:0:65741:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764815.646419:0:65741:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200008:1.0:1775764815.646621:0:65741:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 0, count: 4194304 00000080:00200008:1.0:1775764815.647166:0:65741:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 16, count: 4194304 00000080:00200008:1.0:1775764815.647293:0:65741:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100000 00010000:00010000:1.0:1775764815.647312:0:65741:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764815.771855:0:65751:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764815.771886:0:65751:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764815.771906:0:65751:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764815.771923:0:65751:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764815.771932:0:65751:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764815.771940:0:65751:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764815.771944:0:65751:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c06a 00000080:00010000:1.0:1775764815.771952:0:65751:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c06a from it ffffa60b01fc7bb0 00010000:00010000:1.0:1775764815.771956:0:65751:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1775764815.771967:0:65751:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764815.771984:0:65751:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764815.771991:0:65751:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:1.0:1775764815.772019:0:65751:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7fff58a2b590 00000080:00200008:1.0:1775764815.772049:0:65751:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00010000:00010000:1.0:1775764815.801689:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1775764815.877527:0:65757:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764815.877543:0:65757:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764815.877566:0:65757:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764815.877586:0:65757:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764815.877602:0:65757:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1775764815.877610:0:65757:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764815.877618:0:65757:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764815.877621:0:65757:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c06a 00000080:00010000:2.0:1775764815.877628:0:65757:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c06a from it ffffa60b01fb7cb8 00010000:00010000:2.0:1775764815.877631:0:65757:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764815.877640:0:65757:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1775764815.877667:0:65757:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764815.877673:0:65757:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764815.877679:0:65757:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1775764815.877687:0:65757:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1775764815.877694:0:65757:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764815.877698:0:65757:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764815.877700:0:65757:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c06a 00000080:00010000:2.0:1775764815.877704:0:65757:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c06a from it ffffa60b01fb7b58 00010000:00010000:2.0:1775764815.877707:0:65757:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1775764815.877714:0:65757:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1775764815.877722:0:65757:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764815.877728:0:65757:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=8 00000080:00200008:2.0:1775764815.877734:0:65757:0:(file.c:6226:ll_getattr_dentry()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764815.877737:0:65757:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88),name=f26.sanity-pcc 00000002:00010000:2.0:1775764815.877748:0:65757:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764815.877766:0:65757:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764815.877775:0:65757:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764815.877781:0:65757:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764815.877785:0:65757:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00010000:2.0:1775764815.877787:0:65757:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c063 00000080:00010000:2.0:1775764815.877791:0:65757:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c063 from it ffffa60b01fb7cb8 00010000:00010000:2.0:1775764815.877793:0:65757:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764815.877800:0:65757:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764815.877828:0:65757:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764815.877835:0:65757:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764815.877841:0:65757:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1775764815.877848:0:65757:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1775764815.877853:0:65757:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764815.877856:0:65757:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764815.877859:0:65757:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c06a 00000080:00010000:2.0:1775764815.877862:0:65757:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c06a from it ffffa60b01fb7bf8 00010000:00010000:2.0:1775764815.877864:0:65757:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1775764815.877871:0:65757:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c06a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527c5 expref: -99 pid: 65717 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:2.0:1775764815.877877:0:65757:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764815.877881:0:65757:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=265 00000080:00200000:2.0:1775764815.877885:0:65757:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88), inode mode 81a4 mask 46 00000080:00200008:2.0:1775764815.877890:0:65757:0:(file.c:1057:ll_file_open()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 154402 00000080:00200080:2.0:1775764815.877916:0:65757:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88) cmd=c00466fc arg=7ffe773417dc 00000080:00200008:2.0:1775764815.877976:0:65757:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 154002 00000080:00200008:2.0:1775764816.041608:0:65690:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100002 00000002:00100000:2.0:1775764816.041708:0:65690:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b2382dadf80 x1862022098664064/t34359739274(34359739274) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764828 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764816.041721:0:65690:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764816.041726:0:65690:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5c00 x1862022098664704/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764816.041763:0:65690:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:65690:1862022098664704:192.168.202.128@tcp:35:multiop.0 00000100:00100000:2.0:1775764816.041878:0:65690:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0faa600 going to sleep for 11 seconds 00000100:00100000:1.0:1775764816.738849:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119068800 00000100:00100000:1.0:1775764816.738862:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119068800 00000100:00100000:1.0:1775764816.738973:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 262 00000100:00100000:1.0:1775764816.738985:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dadc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:44493:x1862022119068800:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764816.739006:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764816.739336:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764816.739343:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dadc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:44493:x1862022119068800:12345-192.168.202.128@tcp:104: Request processed in 361us (1564us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764816.739351:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 262 00010000:00010000:2.0:1775764816.739403:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764816.739410:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764816.739420:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2381497400 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764816.739424:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764816.739432:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764816.739441:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764816.739482:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764816.739492:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764816.739534:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764816.739542:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764816.739571:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764816.739579:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c063 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e030527b0 expref: -99 pid: 65690 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764816.739596:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764816.739597:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764816.739599:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764816.761195:0:65803:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:65803:1862022098664960:192.168.202.128@tcp:101:multiop.0 00010000:00010000:1.0:1775764816.761252:0:65803:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c078 lrc: 4/0,1 mode: --/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527e8 expref: -99 pid: 65803 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764816.761266:0:65803:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c078 lrc: 4/0,1 mode: --/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527e8 expref: -99 pid: 65803 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764816.761280:0:65803:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c078 lrc: 4/0,1 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527e8 expref: -99 pid: 65803 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764816.761292:0:65803:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c078 lrc: 4/0,1 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527e8 expref: -99 pid: 65803 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764816.761308:0:65803:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=200002b, status=0 req@ffff8b2382daca80 x1862022098664960/t34359739278(34359739278) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764832 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764816.761322:0:65803:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382daca80 x1862022098664960/t34359739278(34359739278) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764832 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764816.761337:0:65803:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c078 lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527e8 expref: -99 pid: 65803 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764816.761348:0:65803:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 8 res 8589949620/18 (0 0) 00000080:00200000:1.0:1775764816.761354:0:65803:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1775764816.761516:0:65803:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c078 00000080:00010000:1.0:1775764816.761524:0:65803:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c078 from it ffffa60b0200fbf0 00010000:00010000:1.0:1775764816.761529:0:65803:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c078 lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527e8 expref: -99 pid: 65803 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764816.761541:0:65803:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c078 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527e8 expref: -99 pid: 65803 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764816.761570:0:65803:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382daca80 x1862022098664960/t34359739278(34359739278) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764832 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00010000:00010000:1.0:1775764816.825825:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764816.952320:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2379:1862022098665344:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764816.952434:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098665600:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764816.952492:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1775764816.952498:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2379:1862022098665216:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764816.952522:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098665472:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764816.955433:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2379:1862022098665344:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764816.962886:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2379:1862022098665216:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764816.964136:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098665600:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764816.964429:0:2379:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b2389398000: committing for last_committed 34359738457 gen 1 00000100:00100000:2.0:1775764816.964443:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098665472:192.168.202.128@tcp:400:kworker.0 00010000:00010000:2.0:1775764817.847673:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764818.871716:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764819.731055:0:65830:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 34359739278 gen 1 00000100:00100000:2.0:1775764819.731077:0:65830:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23b9ea6300 x1862022098664448/t34359739275(34359739275) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764831 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764819.731094:0:65830:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b2382daca80 x1862022098664960/t34359739278(34359739278) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764832 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764819.731104:0:65830:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:65830:1862022098665728:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764819.731130:0:65830:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764819.731144:0:65830:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764819.731157:0:65830:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764819.731168:0:65830:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764819.731182:0:65830:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764819.731191:0:65830:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764819.731198:0:65830:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764819.731210:0:65830:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b2382dac000 x1862022098665728/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764835 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764819.731229:0:65830:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764819.731243:0:65830:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764819.731250:0:65830:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764819.731282:0:65830:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764819.731288:0:65830:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c07f 00000080:00010000:2.0:1775764819.731294:0:65830:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764819.731297:0:65830:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c07f from it ffffa60b0218fcb8 00010000:00010000:2.0:1775764819.731300:0:65830:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764819.731308:0:65830:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764819.731371:0:65830:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764819.731388:0:65830:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764819.731397:0:65830:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764819.731408:0:65830:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764819.731414:0:65830:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764819.731419:0:65830:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764819.731421:0:65830:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c07f 00000080:00010000:2.0:1775764819.731425:0:65830:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c07f from it ffffa60b0218fb58 00010000:00010000:2.0:1775764819.731428:0:65830:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764819.731435:0:65830:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764819.731443:0:65830:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764819.731512:0:65830:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=f26.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), flags=8 00000080:00200000:2.0:1775764819.731517:0:65830:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f26.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=getattr 00000002:00010000:2.0:1775764819.731530:0:65830:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f26.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764819.731569:0:65830:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65830 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764819.731577:0:65830:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65830 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764819.731586:0:65830:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65830 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764819.731609:0:65830:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:65830:1862022098665856:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764819.731695:0:65830:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0faacc0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764819.743435:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119069696 00000100:00100000:1.0:1775764819.743446:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119069696 00000100:00100000:1.0:1775764819.743535:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 263 00000100:00100000:1.0:1775764819.743544:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dae680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119069696:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764819.743566:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c078 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e030527e8 expref: -99 pid: 65803 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764819.743657:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764819.743664:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dae680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119069696:12345-192.168.202.128@tcp:104: Request processed in 122us (348us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764819.743671:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 263 00010000:00010000:2.0:1775764819.744059:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764819.744066:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c078 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e030527e8 expref: -99 pid: 65803 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764819.744076:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2381496000 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764819.744081:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c078 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e030527e8 expref: -99 pid: 65803 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764819.744093:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c078 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e030527e8 expref: -99 pid: 65803 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764819.744101:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c078 lrc: 3/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x57b8ff0e030527e8 expref: -99 pid: 65803 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764819.744130:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764819.744170:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c078 lrc: 2/0,0 mode: --/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030527e8 expref: -99 pid: 65803 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764819.744177:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764819.744204:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c078 lrc: 1/0,0 mode: --/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030527e8 expref: -99 pid: 65803 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764819.744211:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c078 lrc: 0/0,0 mode: --/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e030527e8 expref: -99 pid: 65803 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764819.744227:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764819.744229:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764819.744231:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764819.744238:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764819.744253:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098665984:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764819.765807:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098665984:192.168.202.128@tcp:103:ldlm_bl.0 00000080:00200008:1.0:1775764819.880086:0:65836:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764819.880099:0:65836:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764819.880122:0:65836:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764819.880142:0:65836:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764819.880160:0:65836:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764819.880168:0:65836:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764819.880178:0:65836:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764819.880181:0:65836:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c07f 00000080:00010000:1.0:1775764819.880188:0:65836:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c07f from it ffffa60b01fefcb8 00010000:00010000:1.0:1775764819.880192:0:65836:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764819.880200:0:65836:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764819.880226:0:65836:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764819.880232:0:65836:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764819.880238:0:65836:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764819.880247:0:65836:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764819.880253:0:65836:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764819.880257:0:65836:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764819.880259:0:65836:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c07f 00000080:00010000:1.0:1775764819.880263:0:65836:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c07f from it ffffa60b01fefb58 00010000:00010000:1.0:1775764819.880265:0:65836:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764819.880272:0:65836:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764819.880280:0:65836:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764819.880287:0:65836:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=8 00000080:00200008:1.0:1775764819.880293:0:65836:0:(file.c:6226:ll_getattr_dentry()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764819.880297:0:65836:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88),name=f26.sanity-pcc 00000002:00010000:1.0:1775764819.880309:0:65836:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764819.880317:0:65836:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052819 expref: -99 pid: 65830 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764819.880325:0:65836:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052819 expref: -99 pid: 65830 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764819.880333:0:65836:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764819.880337:0:65836:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00010000:1.0:1775764819.880340:0:65836:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c086 00000080:00010000:1.0:1775764819.880344:0:65836:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c086 from it ffffa60b01fefcb8 00010000:00010000:1.0:1775764819.880348:0:65836:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052819 expref: -99 pid: 65830 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764819.880355:0:65836:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052819 expref: -99 pid: 65830 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764819.881570:0:65836:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1775764819.881586:0:65836:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x12:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764819.881588:0:65836:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1775764819.881592:0:65836:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x12:0x0] 00000008:00010000:1.0:1775764819.881628:0:65836:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764819.881632:0:65836:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c50b0, osc lock ffff8b2382cf6e40, flags 0x400000400001002 00010000:00010000:1.0:1775764819.881657:0:65836:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1775764819.881674:0:65836:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00000020:00200000:1.0:1775764819.881685:0:65836:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764819.881690:0:65836:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1775764819.881700:0:65836:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00000020:00200000:1.0:1775764819.881714:0:65836:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764819.881741:0:65836:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x12:0x0] updating i_size 16 i_blocks 1 00000020:00010000:1.0:1775764819.881746:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764819.881749:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764819.881753:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c50b0 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764819.881757:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764819.881760:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c50b0 (ffff8b2387cba638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764819.882591:0:65836:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/1, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764819.882660:0:65836:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764819.882679:0:65836:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764819.882694:0:65836:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764819.882706:0:65836:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764819.882714:0:65836:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764819.882733:0:65836:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764819.882737:0:65836:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c07f 00000080:00010000:1.0:1775764819.882742:0:65836:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c07f from it ffffa60b01fefbf8 00010000:00010000:1.0:1775764819.882746:0:65836:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764819.882755:0:65836:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764819.883579:0:65836:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764819.883598:0:65836:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=265 00000080:00200000:1.0:1775764819.883607:0:65836:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88), inode mode 81a4 mask 44 00000080:00200008:1.0:1775764819.883617:0:65836:0:(file.c:1057:ll_file_open()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100000 00000080:00200080:1.0:1775764819.883673:0:65836:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88) cmd=400866fc arg=7ffe4e422698 00000080:00200008:1.0:1775764819.885360:0:65836:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 0, count: 1048576 00000080:00200000:1.0:1775764819.885421:0:65836:0:(file.c:2058:ll_file_io_generic()) f26.sanity-pcc: read ppos: 16, bytes: 1048560 00000080:00200000:1.0:1775764819.885427:0:65836:0:(file.c:1815:ll_io_set_mirror()) f26.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764819.885437:0:65836:0:(cl_io.c:214:cl_io_rw_init()) header@000000002eb11a92[0x110, 2, [0x200003ab4:0x12:0x0] hash] 00000020:00200000:1.0:1775764819.885439:0:65836:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [16, 1048576) 0 0 00000080:00200000:1.0:1775764819.885444:0:65836:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764819.886933:0:65836:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x12:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764819.886939:0:65836:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:1.0:1775764819.886951:0:65836:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 16 chunk: [16, 1048576) 1048576, 1048560 00020000:00200000:1.0:1775764819.886956:0:65836:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764819.887003:0:65836:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:1.0:1775764819.887019:0:65836:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764819.887024:0:65836:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c50b0, osc lock ffff8b2382cf6e40, flags 0x400000000000000 00010000:00010000:1.0:1775764819.887048:0:65836:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764819.887065:0:65836:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764819.887078:0:65836:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764819.887087:0:65836:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764819.887094:0:65836:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1775764819.887111:0:65836:0:(vvp_io.c:826:vvp_io_read_start()) f26.sanity-pcc: read [16, 1048576) 00000001:00010000:1.0:1775764819.887126:0:65836:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x12:0x0] 00000008:00010000:1.0:1775764819.887132:0:65836:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764819.887134:0:65836:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5930, osc lock ffff8b2382cf7a20, flags 0x400000400001002 00010000:00010000:1.0:1775764819.887142:0:65836:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764819.887152:0:65836:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00000020:00200000:1.0:1775764819.887162:0:65836:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764819.887165:0:65836:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764819.887173:0:65836:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00000020:00200000:1.0:1775764819.887180:0:65836:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764819.887186:0:65836:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x12:0x0] updating i_size 16 i_blocks 1 00000020:00010000:1.0:1775764819.887190:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764819.887193:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764819.887195:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5930 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764819.887199:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764819.887203:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5930 (ffff8b2387cba638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764819.887215:0:65836:0:(vvp_io.c:885:vvp_io_read_start()) count:1048560 iocb pos:16 00000020:00010000:1.0:1775764819.887227:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764819.887230:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764819.887233:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c50b0 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764819.887237:0:65836:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1775764819.887247:0:65836:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00000020:00010000:1.0:1775764819.887261:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764819.887263:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c50b0 (ffff8b2387cba638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764819.887268:0:65836:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:1.0:1775764819.887292:0:65836:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764819.887296:0:65836:0:(file.c:2206:ll_file_io_generic()) f26.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:1.0:1775764819.887299:0:65836:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:1.0:1775764819.893927:0:65836:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 16, count: 1048576 00000080:00200000:1.0:1775764819.893980:0:65836:0:(file.c:2058:ll_file_io_generic()) f26.sanity-pcc: read ppos: 16, bytes: 1048576 00000080:00200000:1.0:1775764819.893985:0:65836:0:(file.c:1815:ll_io_set_mirror()) f26.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764819.893993:0:65836:0:(cl_io.c:214:cl_io_rw_init()) header@000000002eb11a92[0x110, 2, [0x200003ab4:0x12:0x0] hash] 00000020:00200000:1.0:1775764819.893996:0:65836:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [16, 1048592) 0 0 00000080:00200000:1.0:1775764819.894000:0:65836:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764819.894016:0:65836:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x12:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764819.894019:0:65836:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:1.0:1775764819.894027:0:65836:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 16 chunk: [16, 1048592) 1048592, 1048576 00020000:00200000:1.0:1775764819.894031:0:65836:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764819.894059:0:65836:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 256] 00000008:00010000:1.0:1775764819.894074:0:65836:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764819.894079:0:65836:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c50b0, osc lock ffff8b2382cf6e40, flags 0x400000000000000 00010000:00010000:1.0:1775764819.894102:0:65836:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764819.894123:0:65836:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1052671) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764819.894137:0:65836:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764819.894147:0:65836:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764819.894154:0:65836:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1775764819.894164:0:65836:0:(vvp_io.c:826:vvp_io_read_start()) f26.sanity-pcc: read [16, 1048592) 00000001:00010000:1.0:1775764819.894180:0:65836:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x12:0x0] 00000008:00010000:1.0:1775764819.894186:0:65836:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764819.894188:0:65836:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5930, osc lock ffff8b2382cf7a20, flags 0x400000400001002 00010000:00010000:1.0:1775764819.894195:0:65836:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764819.894203:0:65836:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00000020:00200000:1.0:1775764819.894212:0:65836:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764819.894216:0:65836:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764819.894229:0:65836:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00000020:00200000:1.0:1775764819.894235:0:65836:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764819.894241:0:65836:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x12:0x0] updating i_size 16 i_blocks 1 00000020:00010000:1.0:1775764819.894244:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764819.894247:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764819.894250:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5930 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764819.894254:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764819.894256:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5930 (ffff8b2387cba638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764819.894263:0:65836:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:16 00000020:00010000:1.0:1775764819.894271:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764819.894274:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764819.894276:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c50b0 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764819.894280:0:65836:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764819.894288:0:65836:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 14 lru_type: 1 00000020:00010000:1.0:1775764819.894301:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764819.894304:0:65836:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c50b0 (ffff8b2387cba638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764819.894309:0:65836:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:1.0:1775764819.894332:0:65836:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764819.894335:0:65836:0:(file.c:2206:ll_file_io_generic()) f26.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:1.0:1775764819.894338:0:65836:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:1.0:1775764819.895709:0:65836:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100000 00010000:00010000:1.0:1775764819.895750:0:65836:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764820.023153:0:65846:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764820.023185:0:65846:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764820.023206:0:65846:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764820.023223:0:65846:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764820.023231:0:65846:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764820.023239:0:65846:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764820.023243:0:65846:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c07f 00000080:00010000:2.0:1775764820.023249:0:65846:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c07f from it ffffa60b021afbb0 00010000:00010000:2.0:1775764820.023253:0:65846:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764820.023262:0:65846:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764820.023276:0:65846:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764820.023284:0:65846:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764820.023410:0:65846:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7fff7de6bb20 00000080:00200008:2.0:1775764820.023446:0:65846:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:2.0:1775764820.181162:0:65853:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764820.181190:0:65853:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764820.181209:0:65853:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764820.181227:0:65853:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764820.181241:0:65853:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764820.181249:0:65853:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764820.181252:0:65853:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c07f 00000080:00010000:2.0:1775764820.181260:0:65853:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c07f from it ffffa60b01fefbf8 00010000:00010000:2.0:1775764820.181264:0:65853:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764820.181273:0:65853:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c07f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030527fd expref: -99 pid: 65830 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1775764820.181289:0:65853:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764820.181296:0:65853:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=265 00000080:00200000:2.0:1775764820.181302:0:65853:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88), inode mode 81a4 mask 44 00000080:00200008:2.0:1775764820.181307:0:65853:0:(file.c:1057:ll_file_open()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100000 00000080:00200008:2.0:1775764820.181333:0:65853:0:(file.c:6226:ll_getattr_dentry()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764820.181338:0:65853:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88),name=f26.sanity-pcc 00000002:00010000:2.0:1775764820.181349:0:65853:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764820.181357:0:65853:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052819 expref: -99 pid: 65830 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764820.181366:0:65853:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052819 expref: -99 pid: 65830 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764820.181372:0:65853:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764820.181376:0:65853:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00010000:2.0:1775764820.181378:0:65853:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c086 00000080:00010000:2.0:1775764820.181382:0:65853:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c086 from it ffffa60b01fefcd8 00010000:00010000:2.0:1775764820.181385:0:65853:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052819 expref: -99 pid: 65830 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764820.181392:0:65853:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052819 expref: -99 pid: 65830 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200008:2.0:1775764820.181443:0:65853:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 0, count: 4194304 00000080:00200008:2.0:1775764820.181543:0:65853:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 16, count: 4194304 00000080:00200008:2.0:1775764820.181605:0:65853:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100000 00010000:00010000:2.0:1775764820.181619:0:65853:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 0 00000080:00200008:1.0:1775764820.582018:0:65803:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100002 00000002:00100000:1.0:1775764820.582121:0:65803:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b2382daca80 x1862022098664960/t34359739278(34359739278) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764832 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764820.582135:0:65803:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764820.582141:0:65803:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd880 x1862022098666240/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764820.582167:0:65803:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:65803:1862022098666240:192.168.202.128@tcp:35:multiop.0 00000100:00100000:1.0:1775764820.582270:0:65803:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1c480 going to sleep for 11 seconds 00010000:00010000:2.0:1775764820.919605:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764821.233290:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764821.233332:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119069952 00000100:00100000:1.0:1775764821.233343:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119069952 00000100:00100000:1.0:1775764821.233397:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 264 00000100:00100000:1.0:1775764821.233408:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23864fce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42325:x1862022119069952:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764821.233426:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03052819 expref: -99 pid: 65830 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764821.233538:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764821.233545:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23864fce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42325:x1862022119069952:12345-192.168.202.128@tcp:104: Request processed in 140us (265us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764821.233552:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 264 00010000:00010000:2.0:1775764821.233782:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764821.233789:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03052819 expref: -99 pid: 65830 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764821.233797:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2380031200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764821.233802:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052819 expref: -99 pid: 65830 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764821.233811:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052819 expref: -99 pid: 65830 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764821.233819:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03052819 expref: -99 pid: 65830 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764821.233850:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1775764821.233859:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764821.233902:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052819 expref: -99 pid: 65830 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764821.233908:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764821.233940:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052819 expref: -99 pid: 65830 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764821.233947:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c086 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052819 expref: -99 pid: 65830 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764821.233963:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764821.233965:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764821.233966:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764821.233975:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764821.233994:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098666624:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764821.252356:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098666624:192.168.202.128@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764821.943749:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764822.072007:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2376:1862022098666880:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764822.072155:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098667136:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764822.072195:0:2376:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:1.0:1775764822.072202:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2376:1862022098666752:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764822.072227:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098667008:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764822.072249:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098667264:192.168.202.128@tcp:400:kworker.0 00010000:00010000:2.0:1775764822.967621:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1775764824.163349:0:65943:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764824.163369:0:65943:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c094 00000080:00010000:2.0:1775764824.163378:0:65943:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764824.163380:0:65943:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c094 from it ffffa60b021dfcb8 00010000:00010000:2.0:1775764824.163387:0:65943:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.163401:0:65943:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764824.164118:0:65943:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764824.164144:0:65943:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764824.164162:0:65943:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764824.164175:0:65943:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764824.164182:0:65943:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764824.164199:0:65943:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764824.164202:0:65943:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c094 00000080:00010000:2.0:1775764824.164206:0:65943:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c094 from it ffffa60b021dfb58 00010000:00010000:2.0:1775764824.164209:0:65943:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764824.164217:0:65943:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764824.164227:0:65943:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764824.164246:0:65943:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=f26.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), flags=8 00000080:00200000:2.0:1775764824.164250:0:65943:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f26.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=getattr 00000002:00010000:2.0:1775764824.164263:0:65943:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f26.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764824.164708:0:65943:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65943 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.164718:0:65943:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65943 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.164728:0:65943:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65943 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764824.164757:0:65943:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:65943:1862022098667520:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764824.164924:0:65943:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23a0faa840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764824.177600:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764824.178299:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764824.178307:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c08d lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03052827 expref: -99 pid: 65915 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764824.178315:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2381496a00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764824.178320:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c08d lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03052827 expref: -99 pid: 65915 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764824.178329:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c08d lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03052827 expref: -99 pid: 65915 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764824.178337:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c08d lrc: 3/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x57b8ff0e03052827 expref: -99 pid: 65915 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764824.178367:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 3 lru_score: 11 lru_type: 0 00010000:00010000:2.0:1775764824.178410:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c08d lrc: 2/0,0 mode: --/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03052827 expref: -99 pid: 65915 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764824.178416:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764824.178439:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c08d lrc: 1/0,0 mode: --/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03052827 expref: -99 pid: 65915 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764824.178447:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c08d lrc: 0/0,0 mode: --/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03052827 expref: -99 pid: 65915 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764824.178503:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764824.178505:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764824.178507:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764824.204411:0:65943:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:65943:1862022098667520:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764824.204445:0:65943:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.204493:0:65943:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.204502:0:65943:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.204514:0:65943:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.204525:0:65943:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.204539:0:65943:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.204546:0:65943:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.204553:0:65943:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764824.204577:0:65943:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea5880 x1862022098667520/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/840 e 0 to 0 dl 1775764840 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764824.204589:0:65943:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764824.204605:0:65943:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.204616:0:65943:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/18 (0 0) 00000080:00200000:2.0:1775764824.204622:0:65943:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764824.204729:0:65943:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0x12:0x0](ffff8b238990db88): rc = 0 00000080:00200000:2.0:1775764824.204750:0:65943:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x12:0x0]: layout version change: 2 -> 2 00000080:00010000:2.0:1775764824.204760:0:65943:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c09b 00000080:00010000:2.0:1775764824.204775:0:65943:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00010000:2.0:1775764824.204778:0:65943:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c09b from it ffffa60b021dfb20 00010000:00010000:2.0:1775764824.204782:0:65943:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.204795:0:65943:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1775764824.204847:0:65943:0:(file.c:6226:ll_getattr_dentry()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764824.204855:0:65943:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88),name=f26.sanity-pcc 00000002:00010000:2.0:1775764824.204882:0:65943:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764824.204892:0:65943:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764824.204902:0:65943:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764824.204909:0:65943:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764824.204913:0:65943:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00010000:2.0:1775764824.204916:0:65943:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c09b 00000080:00010000:2.0:1775764824.204919:0:65943:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c09b from it ffffa60b021dfcb8 00010000:00010000:2.0:1775764824.204922:0:65943:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764824.204929:0:65943:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764824.204982:0:65943:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764824.204990:0:65943:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764824.204997:0:65943:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764824.205009:0:65943:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764824.205015:0:65943:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764824.205019:0:65943:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764824.205113:0:65943:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c094 00000080:00010000:2.0:1775764824.205118:0:65943:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c094 from it ffffa60b021dfbf8 00010000:00010000:2.0:1775764824.205121:0:65943:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764824.205128:0:65943:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764824.205136:0:65943:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764824.205143:0:65943:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=265 00000080:00200000:2.0:1775764824.205149:0:65943:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88), inode mode 81a4 mask 46 00000080:00200008:2.0:1775764824.205157:0:65943:0:(file.c:1057:ll_file_open()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 154402 00000080:00200080:2.0:1775764824.205196:0:65943:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88) cmd=c00466fc arg=7ffd256c57fc 00000080:00200008:2.0:1775764824.205257:0:65943:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 154002 00000080:00200008:1.0:1775764824.274736:0:65949:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764824.274753:0:65949:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764824.274778:0:65949:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764824.274798:0:65949:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764824.274814:0:65949:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764824.274822:0:65949:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764824.274831:0:65949:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764824.274835:0:65949:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c094 00000080:00010000:1.0:1775764824.274842:0:65949:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c094 from it ffffa60b02267cb8 00010000:00010000:1.0:1775764824.274845:0:65949:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764824.274854:0:65949:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764824.274884:0:65949:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764824.274892:0:65949:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764824.274897:0:65949:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764824.274905:0:65949:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764824.274911:0:65949:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764824.274914:0:65949:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764824.274917:0:65949:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c094 00000080:00010000:1.0:1775764824.274920:0:65949:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c094 from it ffffa60b02267b58 00010000:00010000:1.0:1775764824.274922:0:65949:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764824.274929:0:65949:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764824.274936:0:65949:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764824.274944:0:65949:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=8 00000080:00200008:1.0:1775764824.274950:0:65949:0:(file.c:6226:ll_getattr_dentry()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764824.274953:0:65949:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88),name=f26.sanity-pcc 00000002:00010000:1.0:1775764824.274965:0:65949:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764824.274973:0:65949:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764824.274981:0:65949:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764824.274992:0:65949:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764824.274995:0:65949:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00010000:1.0:1775764824.274997:0:65949:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c09b 00000080:00010000:1.0:1775764824.275001:0:65949:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c09b from it ffffa60b02267cb8 00010000:00010000:1.0:1775764824.275003:0:65949:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764824.275011:0:65949:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764824.275055:0:65949:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1775764824.275068:0:65949:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x12:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764824.275071:0:65949:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1775764824.275075:0:65949:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x12:0x0] 00000008:00010000:1.0:1775764824.275182:0:65949:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764824.275186:0:65949:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c50b0, osc lock ffff8b2382cf6e40, flags 0x400000400001002 00010000:00010000:1.0:1775764824.275204:0:65949:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764824.275216:0:65949:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00000020:00200000:1.0:1775764824.275228:0:65949:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764824.275231:0:65949:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764824.275243:0:65949:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00000020:00200000:1.0:1775764824.275254:0:65949:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764824.275267:0:65949:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x12:0x0] updating i_size 16 i_blocks 1 00000020:00010000:1.0:1775764824.275271:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764824.275274:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764824.275276:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c50b0 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764824.275280:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764824.275283:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c50b0 (ffff8b2387cba638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764824.275399:0:65949:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/1, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764824.275430:0:65949:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764824.275438:0:65949:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764824.275445:0:65949:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764824.275572:0:65949:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764824.275580:0:65949:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764824.275585:0:65949:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764824.275587:0:65949:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c094 00000080:00010000:1.0:1775764824.275591:0:65949:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c094 from it ffffa60b02267bf8 00010000:00010000:1.0:1775764824.275594:0:65949:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764824.275605:0:65949:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764824.275615:0:65949:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764824.275620:0:65949:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=265 00000080:00200000:1.0:1775764824.275625:0:65949:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88), inode mode 81a4 mask 44 00000080:00200008:1.0:1775764824.275631:0:65949:0:(file.c:1057:ll_file_open()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100000 00000080:00200080:1.0:1775764824.275663:0:65949:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88) cmd=400866fc arg=7ffd02a66dd8 00000080:00200008:1.0:1775764824.276652:0:65949:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 0, count: 1048576 00000080:00200000:1.0:1775764824.276706:0:65949:0:(file.c:2058:ll_file_io_generic()) f26.sanity-pcc: read ppos: 16, bytes: 1048560 00000080:00200000:1.0:1775764824.276712:0:65949:0:(file.c:1815:ll_io_set_mirror()) f26.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764824.276719:0:65949:0:(cl_io.c:214:cl_io_rw_init()) header@000000002eb11a92[0x110, 2, [0x200003ab4:0x12:0x0] hash] 00000020:00200000:1.0:1775764824.276721:0:65949:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [16, 1048576) 0 0 00000080:00200000:1.0:1775764824.276725:0:65949:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764824.276739:0:65949:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x12:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764824.276742:0:65949:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:1.0:1775764824.276751:0:65949:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 16 chunk: [16, 1048576) 1048576, 1048560 00020000:00200000:1.0:1775764824.276755:0:65949:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764824.276783:0:65949:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:1.0:1775764824.276798:0:65949:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764824.276804:0:65949:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c50b0, osc lock ffff8b2382cf6e40, flags 0x400000000000000 00010000:00010000:1.0:1775764824.276826:0:65949:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:1.0:1775764824.276839:0:65949:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:1.0:1775764824.276851:0:65949:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:1.0:1775764824.276861:0:65949:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:1.0:1775764824.276869:0:65949:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00000080:00200000:1.0:1775764824.276878:0:65949:0:(vvp_io.c:826:vvp_io_read_start()) f26.sanity-pcc: read [16, 1048576) 00000001:00010000:1.0:1775764824.276894:0:65949:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x12:0x0] 00000008:00010000:1.0:1775764824.276899:0:65949:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764824.276901:0:65949:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5930, osc lock ffff8b2382cf7a20, flags 0x400000400001002 00010000:00010000:1.0:1775764824.276908:0:65949:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:1.0:1775764824.276917:0:65949:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00000020:00200000:1.0:1775764824.276926:0:65949:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764824.276929:0:65949:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:1.0:1775764824.276937:0:65949:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00000020:00200000:1.0:1775764824.276943:0:65949:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764824.276948:0:65949:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x12:0x0] updating i_size 16 i_blocks 1 00000020:00010000:1.0:1775764824.276951:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764824.276955:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764824.276957:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5930 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764824.276961:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764824.276964:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5930 (ffff8b2387cba638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764824.276972:0:65949:0:(vvp_io.c:885:vvp_io_read_start()) count:1048560 iocb pos:16 00000020:00010000:1.0:1775764824.276980:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764824.276982:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764824.276985:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c50b0 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764824.276988:0:65949:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:1.0:1775764824.276997:0:65949:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00000020:00010000:1.0:1775764824.277007:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764824.277009:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c50b0 (ffff8b2387cba638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764824.277014:0:65949:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:1.0:1775764824.277035:0:65949:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764824.277038:0:65949:0:(file.c:2206:ll_file_io_generic()) f26.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:1.0:1775764824.277041:0:65949:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:1.0:1775764824.277276:0:65949:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 16, count: 1048576 00000080:00200000:1.0:1775764824.277373:0:65949:0:(file.c:2058:ll_file_io_generic()) f26.sanity-pcc: read ppos: 16, bytes: 1048576 00000080:00200000:1.0:1775764824.277377:0:65949:0:(file.c:1815:ll_io_set_mirror()) f26.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764824.277382:0:65949:0:(cl_io.c:214:cl_io_rw_init()) header@000000002eb11a92[0x110, 2, [0x200003ab4:0x12:0x0] hash] 00000020:00200000:1.0:1775764824.277383:0:65949:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [16, 1048592) 0 0 00000080:00200000:1.0:1775764824.277386:0:65949:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764824.277393:0:65949:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x12:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1775764824.277396:0:65949:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:1.0:1775764824.277399:0:65949:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 16 chunk: [16, 1048592) 1048592, 1048576 00020000:00200000:1.0:1775764824.277402:0:65949:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764824.277418:0:65949:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 256] 00000008:00010000:1.0:1775764824.277424:0:65949:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764824.277427:0:65949:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c50b0, osc lock ffff8b2382cf6e40, flags 0x400000000000000 00010000:00010000:1.0:1775764824.277435:0:65949:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1775764824.277446:0:65949:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1052671) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1775764824.277571:0:65949:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1775764824.277580:0:65949:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1775764824.277586:0:65949:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00000080:00200000:1.0:1775764824.277594:0:65949:0:(vvp_io.c:826:vvp_io_read_start()) f26.sanity-pcc: read [16, 1048592) 00000001:00010000:1.0:1775764824.277602:0:65949:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x12:0x0] 00000008:00010000:1.0:1775764824.277608:0:65949:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764824.277609:0:65949:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5930, osc lock ffff8b2382cf7a20, flags 0x400000400001002 00010000:00010000:1.0:1775764824.277617:0:65949:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1775764824.277626:0:65949:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00000020:00200000:1.0:1775764824.277640:0:65949:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764824.277643:0:65949:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1775764824.277650:0:65949:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00000020:00200000:1.0:1775764824.277655:0:65949:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764824.277659:0:65949:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x12:0x0] updating i_size 16 i_blocks 1 00000020:00010000:1.0:1775764824.277661:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764824.277664:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764824.277666:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5930 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764824.277669:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1a910 (ffff8b2387cba638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764824.277671:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5930 (ffff8b2387cba638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764824.277685:0:65949:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:16 00000020:00010000:1.0:1775764824.277692:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_release():245 00000020:00010000:1.0:1775764824.277694:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764824.277696:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c50b0 (ffff8b2387cbac68/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764824.277700:0:65949:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1775764824.277707:0:65949:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00000020:00010000:1.0:1775764824.277715:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb74e8 (ffff8b2387cba638/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764824.277718:0:65949:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c50b0 (ffff8b2387cba638/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764824.277722:0:65949:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:1.0:1775764824.277737:0:65949:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764824.277740:0:65949:0:(file.c:2206:ll_file_io_generic()) f26.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:1.0:1775764824.277743:0:65949:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:1.0:1775764824.277915:0:65949:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100000 00010000:00010000:1.0:1775764824.277933:0:65949:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 3 lru_score: 12 lru_type: 0 00000080:00200000:2.0:1775764824.352326:0:65959:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764824.352357:0:65959:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764824.352378:0:65959:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764824.352396:0:65959:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764824.352411:0:65959:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764824.352420:0:65959:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764824.352423:0:65959:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c094 00000080:00010000:2.0:1775764824.352431:0:65959:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c094 from it ffffa60b022afbb0 00010000:00010000:2.0:1775764824.352436:0:65959:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764824.352445:0:65959:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764824.352854:0:65959:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764824.352863:0:65959:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764824.352891:0:65959:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7fff24153140 00000080:00200008:2.0:1775764824.352923:0:65959:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:1.0:1775764824.437654:0:65966:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764824.437769:0:65966:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764824.437790:0:65966:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764824.437809:0:65966:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764824.437817:0:65966:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764824.437828:0:65966:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764824.437831:0:65966:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c094 00000080:00010000:1.0:1775764824.437838:0:65966:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c094 from it ffffa60b021e7bf8 00010000:00010000:1.0:1775764824.437841:0:65966:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764824.437849:0:65966:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1775764824.437865:0:65966:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764824.437872:0:65966:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=265 00000080:00200000:1.0:1775764824.437878:0:65966:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88), inode mode 81a4 mask 44 00000080:00200008:1.0:1775764824.437885:0:65966:0:(file.c:1057:ll_file_open()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100000 00000080:00200008:1.0:1775764824.437911:0:65966:0:(file.c:6226:ll_getattr_dentry()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764824.437915:0:65966:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88),name=f26.sanity-pcc 00000002:00010000:1.0:1775764824.437927:0:65966:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764824.437935:0:65966:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764824.437945:0:65966:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764824.437951:0:65966:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764824.437955:0:65966:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00010000:1.0:1775764824.437958:0:65966:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c09b 00000080:00010000:1.0:1775764824.437961:0:65966:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c09b from it ffffa60b021e7cd8 00010000:00010000:1.0:1775764824.437964:0:65966:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764824.437971:0:65966:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200008:1.0:1775764824.438028:0:65966:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 0, count: 4194304 00000080:00200008:1.0:1775764824.438180:0:65966:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 16, count: 4194304 00000080:00200008:1.0:1775764824.438257:0:65966:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100000 00010000:00010000:1.0:1775764824.438287:0:65966:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 3 lru_score: 13 lru_type: 1 00000080:00200008:1.0:1775764824.843706:0:65996:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764824.843722:0:65996:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764824.843746:0:65996:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764824.843765:0:65996:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764824.843782:0:65996:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1775764824.843789:0:65996:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764824.843799:0:65996:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764824.843802:0:65996:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c094 00000080:00010000:1.0:1775764824.843810:0:65996:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c094 from it ffffa60b022efcb8 00010000:00010000:1.0:1775764824.843813:0:65996:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1775764824.843823:0:65996:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764824.843849:0:65996:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764824.843855:0:65996:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764824.843862:0:65996:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764824.843870:0:65996:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764824.843876:0:65996:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764824.843880:0:65996:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764824.843883:0:65996:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c094 00000080:00010000:1.0:1775764824.843886:0:65996:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c094 from it ffffa60b022efb58 00010000:00010000:1.0:1775764824.843889:0:65996:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764824.843894:0:65996:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764824.843902:0:65996:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764824.843908:0:65996:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=8 00000080:00200008:1.0:1775764824.843913:0:65996:0:(file.c:6226:ll_getattr_dentry()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764824.843917:0:65996:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88),name=f26.sanity-pcc 00000002:00010000:1.0:1775764824.843927:0:65996:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764824.843934:0:65996:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764824.843941:0:65996:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764824.843947:0:65996:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764824.843950:0:65996:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00010000:1.0:1775764824.843952:0:65996:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c09b 00000080:00010000:1.0:1775764824.843956:0:65996:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c09b from it ffffa60b022efcb8 00010000:00010000:1.0:1775764824.843958:0:65996:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764824.843964:0:65996:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764824.843996:0:65996:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764824.844002:0:65996:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764824.844008:0:65996:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764824.844015:0:65996:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00000080:00200000:1.0:1775764824.844020:0:65996:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764824.844024:0:65996:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764824.844026:0:65996:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c094 00000080:00010000:1.0:1775764824.844029:0:65996:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c094 from it ffffa60b022efbf8 00010000:00010000:1.0:1775764824.844032:0:65996:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:1.0:1775764824.844039:0:65996:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 0 00000080:00200000:1.0:1775764824.844045:0:65996:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764824.844049:0:65996:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=265 00000080:00200000:1.0:1775764824.844053:0:65996:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88), inode mode 81a4 mask 46 00000080:00200008:1.0:1775764824.844058:0:65996:0:(file.c:1057:ll_file_open()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 154402 00000002:00010000:1.0:1775764824.844070:0:65996:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240114403 00010000:00010000:1.0:1775764824.844114:0:65996:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65996 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764824.844120:0:65996:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65996 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764824.844130:0:65996:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764824.844141:0:65996:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fca80 x1862022098668160/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764824.844150:0:65996:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65996 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764824.844169:0:65996:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:65996:1862022098668160:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764824.844338:0:65996:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1c840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764824.850412:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764824.851161:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764824.851168:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764824.851178:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2380031200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764824.851183:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764824.851193:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764824.851202:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764824.851232:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 3 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1775764824.851242:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764824.851292:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764824.851299:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764824.851326:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764824.851333:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c09b lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052858 expref: -99 pid: 65943 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764824.851350:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764824.851351:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764824.851353:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764824.854079:0:65996:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:65996:1862022098668160:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764824.854101:0:65996:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CW ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764824.854120:0:65996:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 4/1,0 mode: --/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764824.854128:0:65996:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 4/1,0 mode: --/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764824.854141:0:65996:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 4/1,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764824.854151:0:65996:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 4/1,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764824.854166:0:65996:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 5/1,1 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764824.854175:0:65996:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 5/1,1 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764824.854182:0:65996:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 4/0,1 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764824.854192:0:65996:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23864fca80 x1862022098668160/t34359739282(34359739282) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764840 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764824.854204:0:65996:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fca80 x1862022098668160/t34359739282(34359739282) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764840 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764824.854219:0:65996:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764824.854229:0:65996:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 8 res 8589949620/18 (0 0) 00000080:00200000:1.0:1775764824.854234:0:65996:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1775764824.854317:0:65996:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c0a2 00000080:00010000:1.0:1775764824.854323:0:65996:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c0a2 from it ffffa60b022efbf0 00010000:00010000:1.0:1775764824.854326:0:65996:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764824.854334:0:65996:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764824.854357:0:65996:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fca80 x1862022098668160/t34359739282(34359739282) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764840 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764824.854844:0:65996:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88) cmd=c00466fc arg=7ffe5abda87c 00000080:00200008:1.0:1775764824.854874:0:65996:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 154002 00010000:00010000:1.0:1775764824.854895:0:65996:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:1.0:1775764824.943874:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119071104 00000100:00100000:1.0:1775764824.943886:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119071104 00000100:00100000:1.0:1775764824.943977:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 267 00000100:00100000:1.0:1775764824.943989:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dae680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:44493:x1862022119071104:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764824.944008:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764824.944099:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764824.944106:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dae680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:44493:x1862022119071104:12345-192.168.202.128@tcp:104: Request processed in 121us (328us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764824.944114:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 267 00010000:00010000:2.0:1775764824.944168:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764824.944175:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764824.944182:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8741200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764824.944187:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764824.944194:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764824.944202:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 3/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000002:00100000:2.0:1775764824.944257:0:46150:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fca80 x1862022098668160/t34359739282(34359739282) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764840 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764824.944268:0:46150:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764824.944273:0:46150:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5500 x1862022098668672/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764824.944296:0:46150:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_02:fcf15866-4739-4595-8fe1-44abde7ee9d7:46150:1862022098668672:192.168.202.128@tcp:35:ldlm_bl.0 00000100:00100000:2.0:1775764824.944347:0:46150:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23869426c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764824.949178:0:46150:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_02:fcf15866-4739-4595-8fe1-44abde7ee9d7:46150:1862022098668672:192.168.202.128@tcp:35:ldlm_bl.0 00000002:00100000:2.0:1775764824.949218:0:46150:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fca80 x1862022098668160/t34359739282(34359739282) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764840 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764824.949262:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 3 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1775764824.949314:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 2/0,0 mode: --/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764824.949322:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764824.949352:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 1/0,0 mode: --/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764824.949360:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741200/0x73cfd86cfc17c0a2 lrc: 0/0,0 mode: --/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 4 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03052866 expref: -99 pid: 65996 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764824.949375:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764824.949378:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764824.949380:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764824.959816:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764824.961727:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764824.961739:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c0a9 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03052889 expref: -99 pid: 66002 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764824.961752:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2381496000 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764824.961759:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c0a9 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052889 expref: -99 pid: 66002 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764824.961769:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c0a9 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052889 expref: -99 pid: 66002 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764824.961778:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c0a9 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03052889 expref: -99 pid: 66002 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764824.961806:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c071 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030527d3 expref: -99 pid: 65723 timeout: 0 lvb_type: 3 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1775764824.961816:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c05c lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305277f expref: -99 pid: 65664 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764824.961866:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c0a9 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052889 expref: -99 pid: 66002 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764824.961873:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764824.962076:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c0a9 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052889 expref: -99 pid: 66002 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764824.962085:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c0a9 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052889 expref: -99 pid: 66002 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764824.962103:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764824.962105:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764824.962107:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764824.974497:0:66002:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66002:1862022098669696:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764824.974528:0:66002:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PW ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c0b7 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e030528ac expref: -99 pid: 66002 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.974539:0:66002:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c0b7 lrc: 4/1,0 mode: --/PW res: [0x200003ab4:0x12:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e030528ac expref: -99 pid: 66002 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.974552:0:66002:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c0b7 lrc: 4/1,0 mode: PW/PW res: [0x200003ab4:0x12:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e030528ac expref: -99 pid: 66002 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.974570:0:66002:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c0b7 lrc: 4/1,0 mode: PW/PW res: [0x200003ab4:0x12:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e030528ac expref: -99 pid: 66002 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.974584:0:66002:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c0b7 lrc: 5/1,1 mode: PW/PW res: [0x200003ab4:0x12:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e030528ac expref: -99 pid: 66002 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.974593:0:66002:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c0b7 lrc: 5/1,1 mode: PW/PW res: [0x200003ab4:0x12:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e030528ac expref: -99 pid: 66002 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.974605:0:66002:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c0b7 lrc: 4/0,1 mode: PW/PW res: [0x200003ab4:0x12:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e030528ac expref: -99 pid: 66002 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764824.974617:0:66002:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=600002b, status=0 req@ffff8b2382daf100 x1862022098669696/t34359739286(34359739286) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/864 e 0 to 0 dl 1775764840 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764824.974630:0:66002:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382daf100 x1862022098669696/t34359739286(34359739286) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/864 e 0 to 0 dl 1775764840 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764824.974645:0:66002:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c0b7 lrc: 3/0,1 mode: PW/PW res: [0x200003ab4:0x12:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e030528ac expref: -99 pid: 66002 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.974659:0:66002:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 2 res 8589949620/18 (0 0) 00000080:00200000:2.0:1775764824.974668:0:66002:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab4:0x12:0x0] suppgids 0 0: rc 0 00000002:00100000:2.0:1775764824.974697:0:66002:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382daf100 x1862022098669696/t34359739286(34359739286) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764840 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764824.974705:0:66002:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c0b7 00000080:00010000:2.0:1775764824.974712:0:66002:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c0b7 from it ffffa60b021f7cc0 00010000:00010000:2.0:1775764824.974716:0:66002:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c0b7 lrc: 3/0,1 mode: PW/PW res: [0x200003ab4:0x12:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e030528ac expref: -99 pid: 66002 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764824.974723:0:66002:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c0b7 lrc: 2/0,0 mode: PW/PW res: [0x200003ab4:0x12:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e030528ac expref: -99 pid: 66002 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200080:2.0:1775764824.974941:0:66002:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88) cmd=c00c66f3 arg=9d12a0 00000080:00200008:2.0:1775764824.975375:0:66002:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 0, count: 1048576 00000080:00200000:2.0:1775764824.975432:0:66002:0:(file.c:2058:ll_file_io_generic()) f26.sanity-pcc: read ppos: 16, bytes: 1048560 00000080:00200000:2.0:1775764824.975438:0:66002:0:(file.c:1815:ll_io_set_mirror()) f26.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764824.975447:0:66002:0:(cl_io.c:214:cl_io_rw_init()) header@000000002eb11a92[0x110, 2, [0x200003ab4:0x12:0x0] hash] 00000020:00200000:2.0:1775764824.975449:0:66002:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [16, 1048576) 2048 0 00000080:00200000:2.0:1775764824.975482:0:66002:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764824.975498:0:66002:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x12:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764824.975501:0:66002:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764824.975508:0:66002:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 16 chunk: [16, 1048576) 1048576, 1048560 00020000:00200000:2.0:1775764824.975512:0:66002:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764824.975542:0:66002:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:2.0:1775764824.975558:0:66002:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 1 00000008:00010000:2.0:1775764824.975563:0:66002:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8d30, osc lock ffff8b238a3f8000, flags 0x400000000040000 00010000:00010000:2.0:1775764824.975585:0:66002:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00010000:00010000:2.0:1775764824.975598:0:66002:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00010000:00010000:2.0:1775764824.975610:0:66002:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00010000:00010000:2.0:1775764824.975619:0:66002:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00010000:00010000:2.0:1775764824.975626:0:66002:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00000080:00200000:2.0:1775764824.975635:0:66002:0:(vvp_io.c:826:vvp_io_read_start()) f26.sanity-pcc: read [16, 1048576) 00000001:00010000:2.0:1775764824.975649:0:66002:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x12:0x0] 00000008:00010000:2.0:1775764824.975655:0:66002:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764824.975657:0:66002:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c88b0, osc lock ffff8b238a3f8be0, flags 0x400000400001002 00010000:00010000:2.0:1775764824.975664:0:66002:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00010000:00010000:2.0:1775764824.975685:0:66002:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00000020:00200000:2.0:1775764824.975694:0:66002:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764824.975697:0:66002:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00010000:00010000:2.0:1775764824.975704:0:66002:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00000020:00200000:2.0:1775764824.975711:0:66002:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764824.975718:0:66002:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x12:0x0] updating i_size 16 i_blocks 1 00000020:00010000:2.0:1775764824.975722:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764824.975725:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764824.975727:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c88b0 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764824.975732:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764824.975735:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c88b0 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764824.975743:0:66002:0:(vvp_io.c:885:vvp_io_read_start()) count:1048560 iocb pos:16 00000020:00010000:2.0:1775764824.975752:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764824.975755:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764824.975757:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8d30 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764824.975762:0:66002:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 21 lru_type: 1 00010000:00010000:2.0:1775764824.975777:0:66002:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00000020:00010000:2.0:1775764824.975792:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764824.975794:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8d30 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764824.975800:0:66002:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1775764824.975833:0:66002:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764824.975837:0:66002:0:(file.c:2206:ll_file_io_generic()) f26.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1775764824.975840:0:66002:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:2.0:1775764824.975883:0:66002:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 16, count: 1048576 00000080:00200000:2.0:1775764824.975896:0:66002:0:(file.c:2058:ll_file_io_generic()) f26.sanity-pcc: read ppos: 16, bytes: 1048576 00000080:00200000:2.0:1775764824.975900:0:66002:0:(file.c:1815:ll_io_set_mirror()) f26.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764824.975904:0:66002:0:(cl_io.c:214:cl_io_rw_init()) header@000000002eb11a92[0x110, 2, [0x200003ab4:0x12:0x0] hash] 00000020:00200000:2.0:1775764824.975906:0:66002:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [16, 1048592) 2048 0 00000080:00200000:2.0:1775764824.975909:0:66002:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764824.975914:0:66002:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x12:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764824.975917:0:66002:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764824.975920:0:66002:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 16 chunk: [16, 1048592) 1048592, 1048576 00020000:00200000:2.0:1775764824.975922:0:66002:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764824.975934:0:66002:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 256] 00000008:00010000:2.0:1775764824.975939:0:66002:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 1 00000008:00010000:2.0:1775764824.975942:0:66002:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8d30, osc lock ffff8b238a3f8000, flags 0x400000000040000 00010000:00010000:2.0:1775764824.975950:0:66002:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:2.0:1775764824.975957:0:66002:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1052671) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:2.0:1775764824.975969:0:66002:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:2.0:1775764824.975975:0:66002:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:2.0:1775764824.975981:0:66002:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00000080:00200000:2.0:1775764824.975987:0:66002:0:(vvp_io.c:826:vvp_io_read_start()) f26.sanity-pcc: read [16, 1048592) 00000001:00010000:2.0:1775764824.975994:0:66002:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x12:0x0] 00000008:00010000:2.0:1775764824.975999:0:66002:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764824.976001:0:66002:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c88b0, osc lock ffff8b238a3f8be0, flags 0x400000400001002 00010000:00010000:2.0:1775764824.976006:0:66002:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:2.0:1775764824.976014:0:66002:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00000020:00200000:2.0:1775764824.976020:0:66002:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764824.976023:0:66002:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:2.0:1775764824.976029:0:66002:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00000020:00200000:2.0:1775764824.976039:0:66002:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764824.976043:0:66002:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x12:0x0] updating i_size 16 i_blocks 1 00000020:00010000:2.0:1775764824.976045:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764824.976047:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764824.976049:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c88b0 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764824.976052:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c490 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764824.976054:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c88b0 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764824.976059:0:66002:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:16 00000020:00010000:2.0:1775764824.976064:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764824.976066:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764824.976068:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8d30 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764824.976072:0:66002:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 22 lru_type: 1 00010000:00010000:2.0:1775764824.976079:0:66002:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00000020:00010000:2.0:1775764824.976086:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b4e8 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764824.976088:0:66002:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8d30 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764824.976091:0:66002:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1775764824.976103:0:66002:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764824.976106:0:66002:0:(file.c:2206:ll_file_io_generic()) f26.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1775764824.976109:0:66002:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200000:2.0:1775764824.976283:0:66002:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764824.976289:0:66002:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x12:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764824.976292:0:66002:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764824.976295:0:66002:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:2.0:1775764824.976400:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098669824:192.168.202.128@tcp:1:lfs.0 00000100:00100000:1.0:1775764824.984964:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119071360 00000100:00100000:1.0:1775764824.984978:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119071360 00000100:00100000:1.0:1775764824.985154:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 269 00000100:00100000:1.0:1775764824.985167:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dac380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42946:x1862022119071360:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764824.985188:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:1.0:1775764824.985311:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764824.985318:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dac380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42946:x1862022119071360:12345-192.168.202.128@tcp:104: Request processed in 155us (544us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764824.985326:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 269 00010000:00010000:2.0:1775764824.985375:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764824.985382:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764824.985391:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5fe00 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:2.0:1775764824.985397:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800428400000000 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00000080:00200000:2.0:1775764824.985488:0:46150:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764824.986778:0:46150:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775764824.986897:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764824.986909:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764824.987122:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764824.987131:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000009e189fb3/0x73cfd86cfc17c047 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e03052747 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1775764824.987153:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764824.987155:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764824.987158:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764825.000722:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098669952:192.168.202.128@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764825.015640:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1775764825.637164:0:66028:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764825.637195:0:66028:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764825.637216:0:66028:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764825.637234:0:66028:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1775764825.637241:0:66028:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764825.637250:0:66028:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764825.637253:0:66028:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c094 00000080:00010000:1.0:1775764825.637259:0:66028:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c094 from it ffffa60b0233fbf8 00010000:00010000:1.0:1775764825.637262:0:66028:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764825.637268:0:66028:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:1.0:1775764825.637281:0:66028:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764825.637293:0:66028:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764825.637296:0:66028:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764825.637300:0:66028:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764825.639926:0:66028:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:1.0:1775764825.639945:0:66028:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764825.639958:0:66028:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764825.639962:0:66028:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c094 00000080:00010000:1.0:1775764825.639971:0:66028:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c094 from it ffffa60b0233fb38 00010000:00010000:1.0:1775764825.639975:0:66028:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764825.639985:0:66028:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:1.0:1775764825.640002:0:66028:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:1.0:1775764825.640010:0:66028:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f26.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b238629af00, open_flags 8042, mode 81a4 opened 0 00000080:00200000:1.0:1775764825.640025:0:66028:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f26.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000002:00010000:1.0:1775764825.640053:0:66028:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f26.sanity-pcc,[0x200003ab4:0x15:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0100103 00010000:00010000:1.0:1775764825.640067:0:66028:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:1.0:1775764825.640127:0:66028:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:1.0:1775764825.640135:0:66028:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764825.640137:0:66028:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c094 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305283c expref: -99 pid: 65943 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:1.0:1775764825.640174:0:66028:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66028 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764825.640185:0:66028:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66028 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764825.640197:0:66028:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764825.640209:0:66028:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fc000 x1862022098671104/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764825.640217:0:66028:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66028 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764825.640242:0:66028:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:66028:1862022098671104:192.168.202.128@tcp:101:multiop.0 00000100:00100000:1.0:1775764825.643864:0:66028:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1ccc0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764825.660757:0:66028:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:66028:1862022098671104:192.168.202.128@tcp:101:multiop.0 00010000:00010000:1.0:1775764825.660786:0:66028:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764825.660814:0:66028:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764825.660821:0:66028:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764825.660832:0:66028:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764825.660843:0:66028:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764825.660859:0:66028:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764825.660867:0:66028:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764825.660874:0:66028:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764825.660885:0:66028:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=2b, status=0 req@ffff8b23864fc000 x1862022098671104/t34359739297(34359739297) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/864 e 0 to 0 dl 1775764841 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764825.660899:0:66028:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fc000 x1862022098671104/t34359739297(34359739297) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/864 e 0 to 0 dl 1775764841 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764825.660912:0:66028:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764825.660926:0:66028:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764825.660937:0:66028:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/18 (0 0) 00000080:00200000:1.0:1775764825.660944:0:66028:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764825.661034:0:66028:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0x12:0x0], updating i_size 16 00000080:00200000:1.0:1775764825.661038:0:66028:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0x12:0x0](ffff8b238990db88): rc = 0 00000080:00200000:1.0:1775764825.661057:0:66028:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x12:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1775764825.661067:0:66028:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c0c5 00000080:00010000:1.0:1775764825.661079:0:66028:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00200008:1.0:1775764825.661095:0:66028:0:(file.c:1057:ll_file_open()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100102 00000002:00100000:1.0:1775764825.661115:0:66028:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fc000 x1862022098671104/t34359739297(34359739297) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764841 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1775764825.661159:0:66028:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c0c5 from it ffff8b2389ed61e0 00010000:00010000:1.0:1775764825.661164:0:66028:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764825.661173:0:66028:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764827.063704:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764827.203689:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2377:1862022098671232:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764827.203714:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098671744:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764827.203728:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098671616:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764827.207059:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098671488:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764827.211913:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098671360:192.168.202.128@tcp:400:kworker.0 00000080:00200000:2.0:1775764828.226598:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764828.226614:0:8:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x10:0x0]: flr state: 10, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764828.226618:0:8:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1775764828.226624:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764828.226699:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1775764828.526705:0:66055:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764828.526723:0:66055:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764828.526751:0:66055:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764828.526820:0:66055:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66055 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.526830:0:66055:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66055 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.526840:0:66055:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66055 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764828.526866:0:66055:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66055:1862022098671872:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764828.527037:0:66055:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764828.533984:0:66055:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 34359739297 gen 1 00000100:00100000:2.0:1775764828.534004:0:66055:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23864fc000 x1862022098671104/t34359739297(34359739297) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764841 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764828.534016:0:66055:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66055:1862022098671872:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764828.534040:0:66055:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.534051:0:66055:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.534065:0:66055:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.534076:0:66055:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.534088:0:66055:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.534096:0:66055:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.534104:0:66055:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764828.534113:0:66055:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea6a00 x1862022098671872/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764844 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764828.534136:0:66055:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.534150:0:66055:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764828.534156:0:66055:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764828.534185:0:66055:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764828.534190:0:66055:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c0cc 00000080:00010000:2.0:1775764828.534194:0:66055:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764828.534196:0:66055:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c0cc from it ffffa60b02377cb8 00010000:00010000:2.0:1775764828.534198:0:66055:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.534204:0:66055:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764828.534274:0:66055:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764828.534289:0:66055:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764828.534298:0:66055:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764828.534307:0:66055:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764828.534314:0:66055:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764828.534318:0:66055:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764828.534320:0:66055:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c0cc 00000080:00010000:2.0:1775764828.534323:0:66055:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c0cc from it ffffa60b02377b58 00010000:00010000:2.0:1775764828.534325:0:66055:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764828.534331:0:66055:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764828.534338:0:66055:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764828.534344:0:66055:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=8 00000080:00200008:2.0:1775764828.534352:0:66055:0:(file.c:6226:ll_getattr_dentry()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764828.534356:0:66055:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88),name=f26.sanity-pcc 00000002:00010000:2.0:1775764828.534367:0:66055:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764828.534374:0:66055:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764828.534381:0:66055:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764828.534387:0:66055:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764828.534391:0:66055:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00010000:2.0:1775764828.534393:0:66055:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c0c5 00000080:00010000:2.0:1775764828.534396:0:66055:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c0c5 from it ffffa60b02377cb8 00010000:00010000:2.0:1775764828.534399:0:66055:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764828.534406:0:66055:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764828.534445:0:66055:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764828.534484:0:66055:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764828.534491:0:66055:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764828.534499:0:66055:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764828.534505:0:66055:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764828.534509:0:66055:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764828.534511:0:66055:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c0cc 00000080:00010000:2.0:1775764828.534515:0:66055:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c0cc from it ffffa60b02377bf8 00010000:00010000:2.0:1775764828.534518:0:66055:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764828.534524:0:66055:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764828.534533:0:66055:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764828.534537:0:66055:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=265 00000080:00200000:2.0:1775764828.534541:0:66055:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88), inode mode 81a4 mask 46 00000080:00200008:2.0:1775764828.534551:0:66055:0:(file.c:1057:ll_file_open()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 154402 00000080:00200080:2.0:1775764828.534593:0:66055:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88) cmd=c00466fc arg=7ffe248ed1cc 00800000:00000080:2.0:1775764828.534608:0:66055:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d9 len=56 karg=ffff8b238872e840 uarg=0000000000000000 00000002:00000080:2.0:1775764828.534615:0:66055:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d9 len=56 karg=ffff8b238872e840 uarg=0000000000000000 00010000:00010000:2.0:1775764828.534658:0:66055:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764828.534706:0:66055:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x12:0x0]: losing layout lock 00010000:00010000:2.0:1775764828.534741:0:66055:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17c0be lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e030528c8 expref: -99 pid: 66002 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764828.534753:0:66055:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x12:0x0]: losing layout lock 00010000:00010000:2.0:1775764828.534771:0:66055:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764828.534775:0:66055:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17c0be lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030528c8 expref: -99 pid: 66002 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764828.534779:0:66055:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00000100:00100000:2.0:1775764828.534794:0:66055:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66055:1862022098672128:192.168.202.128@tcp:103:lfs.0 00000100:00100000:2.0:1775764828.534866:0:66055:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764828.537810:0:66055:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66055:1862022098672128:192.168.202.128@tcp:103:lfs.0 00010000:00010000:2.0:1775764828.537832:0:66055:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741400/0x73cfd86cfc17c0c5 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030528f9 expref: -99 pid: 66028 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764828.537843:0:66055:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17c0be lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030528c8 expref: -99 pid: 66002 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000020:00100000:2.0:1775764828.537867:0:66055:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 58, max 7 00000100:00100000:2.0:1775764828.537872:0:66055:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6a00 x1862022098672256/t0(0) o58->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764828.537892:0:66055:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66055:1862022098672256:192.168.202.128@tcp:58:lfs.0 00000100:00100000:2.0:1775764828.537960:0:66055:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764828.554050:0:66055:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66055:1862022098672256:192.168.202.128@tcp:58:lfs.0 00000080:00010000:2.0:1775764828.554116:0:66055:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000002:00010000:2.0:1775764828.554143:0:66055:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200003ab4:0x12:0x0], intent: layout flags 0 00010000:00010000:2.0:1775764828.554207:0:66055:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66055 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.554220:0:66055:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66055 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.554231:0:66055:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66055 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764828.560289:0:66055:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66055:1862022098672384:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764828.560330:0:66055:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.560344:0:66055:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764828.570441:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764828.624084:0:66056:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098673024:192.168.202.128@tcp:53:lhsmtool_posix.0 00000002:00000080:1.0:1775764828.624135:0:66056:0:(mdc_request.c:1758:mdc_ioc_fid2path()) path got [0x200000007:0x1:0x0] from 18446744073709551615 #0: f26.sanity-pcc 00000080:00200000:1.0:1775764828.624240:0:66056:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764828.624269:0:66056:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764828.624755:0:66056:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764828.624781:0:66056:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764828.624789:0:66056:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764828.624800:0:66056:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764828.624804:0:66056:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c0cc 00000080:00010000:1.0:1775764828.624812:0:66056:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c0cc from it ffffa60b02357bf8 00010000:00010000:1.0:1775764828.624816:0:66056:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764828.624825:0:66056:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764828.624844:0:66056:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764828.624872:0:66056:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764828.624882:0:66056:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764828.624889:0:66056:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764828.624898:0:66056:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764828.624905:0:66056:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764828.624909:0:66056:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764828.624912:0:66056:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c0cc 00000080:00010000:1.0:1775764828.624916:0:66056:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c0cc from it ffffa60b02357b38 00010000:00010000:1.0:1775764828.624920:0:66056:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764828.624927:0:66056:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764828.624936:0:66056:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:1.0:1775764828.624941:0:66056:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=. :VOLATILE:0000:672BE8A9:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b2383ec8600, open_flags 280c2, mode 8180 opened 0 00000080:00200000:1.0:1775764828.624949:0:66056:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=. :VOLATILE:0000:672BE8A9:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000002:00010000:1.0:1775764828.624972:0:66056:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: . :VOLATILE:0000:672BE8A9:fd=09,[0x200003ab4:0x16:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0500303 00010000:00010000:1.0:1775764828.624987:0:66056:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764828.625047:0:66056:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764828.625054:0:66056:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764828.625057:0:66056:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c0cc lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052915 expref: -99 pid: 66055 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764828.625099:0:66056:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66056 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.625106:0:66056:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66056 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764828.625118:0:66056:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764828.625144:0:66056:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fea00 x1862022098673280/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1088/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764828.625155:0:66056:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66056 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764828.625181:0:66056:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098673280:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764828.627535:0:66056:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1ccc0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764828.656890:0:66056:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098673280:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:1.0:1775764828.656918:0:66056:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305295b expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.656968:0:66056:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305295b expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.656977:0:66056:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305295b expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.656989:0:66056:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305295b expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.657002:0:66056:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305295b expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.657017:0:66056:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305295b expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.657027:0:66056:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305295b expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.657040:0:66056:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305295b expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764828.657051:0:66056:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8b23864fea00 x1862022098673280/t34359739299(34359739299) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1088/664 e 0 to 0 dl 1775764844 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764828.657064:0:66056:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fea00 x1862022098673280/t34359739299(34359739299) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1088/664 e 0 to 0 dl 1775764844 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764828.657081:0:66056:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305295b expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764828.657095:0:66056:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305295b expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.657106:0:66056:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/22 (0 0) 00000080:00200000:1.0:1775764828.657113:0:66056:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764828.657127:0:66056:0:(namei.c:1620:ll_create_it()) VFS Op:name=. :VOLATILE:0000:672BE8A9:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000080:00200000:1.0:1775764828.657206:0:66056:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab4:0x16:0x0](ffff8b2389a65348) 00000080:00200000:1.0:1775764828.657334:0:66056:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x16:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1775764828.657346:0:66056:0:(namei.c:1586:ll_create_node()) setting l_ast_data to inode [0x200003ab4:0x16:0x0](ffff8b2389a65348) 00000080:00010000:1.0:1775764828.657349:0:66056:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x16:0x0](ffff8b2389a65348) for lock 0x73cfd86cfc17c0e1 00000080:00010000:1.0:1775764828.657359:0:66056:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x16:0x0](ffff8b2389a65348) 00000080:00200008:1.0:1775764828.657372:0:66056:0:(file.c:1057:ll_file_open()) START file . :VOLATILE:0000:672BE8A9:fd=09:[0x200003ab4:0x16:0x0](ffff8b2389a65348), flags 500302 00000002:00100000:1.0:1775764828.657390:0:66056:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fea00 x1862022098673280/t34359739299(34359739299) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 608/664 e 0 to 0 dl 1775764844 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1775764828.657434:0:66056:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c0e1 from it ffff8b2389ed6a80 00010000:00010000:1.0:1775764828.657438:0:66056:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305295b expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.657448:0:66056:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305295b expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764828.657648:0:66056:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x16:0x0](ffff8b2389a65348), inode mode 8180 mask 40 00000080:00200000:1.0:1775764828.657697:0:66056:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764828.657713:0:66056:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764828.657758:0:66056:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66056 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.657765:0:66056:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66056 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.657774:0:66056:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66056 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764828.657795:0:66056:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098673408:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764828.657892:0:66056:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1ca80 going to sleep for 11 seconds 00000100:00100000:1.0:1775764828.665017:0:66056:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098673408:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:1.0:1775764828.665043:0:66056:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052970 expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.665056:0:66056:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052970 expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.665073:0:66056:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052970 expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.665084:0:66056:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052970 expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.665097:0:66056:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052970 expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.665106:0:66056:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052970 expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.665114:0:66056:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052970 expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764828.665138:0:66056:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864fca80 x1862022098673408/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764844 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764828.665153:0:66056:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052970 expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.665166:0:66056:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1775764828.665172:0:66056:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764828.665203:0:66056:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:1.0:1775764828.665209:0:66056:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c0e8 00000080:00010000:1.0:1775764828.665215:0:66056:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764828.665218:0:66056:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c0e8 from it ffffa60b02357c10 00010000:00010000:1.0:1775764828.665221:0:66056:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052970 expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.665230:0:66056:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052970 expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764828.665256:0:66056:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764828.665274:0:66056:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=. :VOLATILE:0000:672BE8A9:fd=09, flags=0 00000080:00200000:1.0:1775764828.665281:0:66056:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764828.665296:0:66056:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764828.665307:0:66056:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052970 expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764828.665317:0:66056:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052970 expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764828.665323:0:66056:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764828.665327:0:66056:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764828.665330:0:66056:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c0e8 00000080:00010000:1.0:1775764828.665333:0:66056:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c0e8 from it ffffa60b02357d70 00010000:00010000:1.0:1775764828.665335:0:66056:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052970 expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764828.665342:0:66056:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052970 expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764828.665349:0:66056:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:1.0:1775764828.665355:0:66056:0:(namei.c:2405:ll_unlink()) VFS Op:name=. :VOLATILE:0000:672BE8A9:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08) 00010000:00010000:1.0:1775764828.665425:0:66056:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052970 expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764828.665681:0:66056:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e0305295b expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764828.665708:0:66056:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x16:0x0]: losing layout lock 00010000:00010000:1.0:1775764828.665757:0:66056:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e0305295b expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764828.665765:0:66056:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052970 expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764828.665770:0:66056:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1775764828.665773:0:66056:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0e1 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e0305295b expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764828.665792:0:66056:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0e8 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052970 expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000020:00100000:1.0:1775764828.665809:0:66056:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1775764828.665816:0:66056:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fca80 x1862022098673664/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/1032 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764828.665841:0:66056:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098673664:192.168.202.128@tcp:36:lhsmtool_posix.0 00000100:00100000:1.0:1775764828.665919:0:66056:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1ca80 going to sleep for 11 seconds 00000100:00100000:1.0:1775764828.673496:0:66056:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098673664:192.168.202.128@tcp:36:lhsmtool_posix.0 00000002:00010000:1.0:1775764828.673600:0:66056:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x0:0x0:0x0]) in obj [0x200003ab4:0x16:0x0], intent: getxattr flags 0 00010000:00010000:1.0:1775764828.673819:0:66056:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0ef lrc: 3/1,0 mode: --/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66056 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.673830:0:66056:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0ef lrc: 3/1,0 mode: --/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66056 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.673842:0:66056:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0ef lrc: 3/1,0 mode: --/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66056 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764828.673865:0:66056:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098673792:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764828.673960:0:66056:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1ca80 going to sleep for 11 seconds 00000100:00100000:1.0:1775764828.680301:0:66056:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098673792:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:1.0:1775764828.680328:0:66056:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0ef lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305298c expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.680344:0:66056:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0ef lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305298c expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.680356:0:66056:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0ef lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305298c expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764828.680370:0:66056:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=80 disposition=0, status=0 req@ffff8b23864fca80 x1862022098673792/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 568/1072 e 0 to 0 dl 1775764844 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764828.680382:0:66056:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0ef lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305298c expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.680396:0:66056:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/22 (0 0) 00000080:00200000:1.0:1775764828.680402:0:66056:0:(namei.c:953:ll_intent_lock()) intent lock 128 on i1 [0x200003ab4:0x16:0x0] suppgids 0 -1: rc 0 00000080:00010000:1.0:1775764828.680429:0:66056:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x16:0x0](ffff8b2389a65348) for lock 0x73cfd86cfc17c0ef 00000080:00010000:1.0:1775764828.680438:0:66056:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c0ef from it ffffa60b02357c28 00010000:00010000:1.0:1775764828.680442:0:66056:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0ef lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305298c expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.680491:0:66056:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c0ef lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x16:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305298c expref: -99 pid: 66056 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:1.0:1775764828.681718:0:66056:0:(llite_lib.c:2274:ll_setattr_raw()) START file . :VOLATILE:0000:672BE8A9:fd=09:[0x200003ab4:0x16:0x0](ffff8b2389a65348) current size 0, valid attrs 1846, mode 8b23, uid 0, gid 0, new size 0, atime 1775764828.000000000, mtime 1775764828.000000000, ctime 1775764828.000000000 00000020:00100000:1.0:1775764828.681785:0:66056:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1775764828.681796:0:66056:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fca80 x1862022098674048/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764828.681824:0:66056:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098674048:192.168.202.128@tcp:36:lhsmtool_posix.0 00000100:00100000:1.0:1775764828.681921:0:66056:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1ca80 going to sleep for 11 seconds 00000100:00100000:1.0:1775764828.687983:0:66056:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098674048:192.168.202.128@tcp:36:lhsmtool_posix.0 00010000:00010000:1.0:1775764828.688043:0:66056:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 31 res 8589949620/22 (0 0) 00000080:00010000:1.0:1775764828.688056:0:66056:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x16:0x0](ffff8b2389a65348) 00000002:00010000:1.0:1775764828.688068:0:66056:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x16:0x0]) in obj [0x200003ab4:0x16:0x0], intent: layout flags 0 00010000:00010000:1.0:1775764828.688118:0:66056:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0f6 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66056 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.688413:0:66056:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0f6 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66056 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.688425:0:66056:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0f6 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66056 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764828.688495:0:66056:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098674176:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764828.688565:0:66056:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1ca80 going to sleep for 11 seconds 00000100:00100000:1.0:1775764828.699067:0:66056:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098674176:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:1.0:1775764828.699101:0:66056:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0f6 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305299a expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.699117:0:66056:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0f6 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305299a expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764828.699145:0:66056:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23864fdc00 x1862022098674176/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/400 e 0 to 0 dl 1775764844 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764828.699156:0:66056:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0f6 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305299a expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764828.699170:0:66056:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0f6 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305299a expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.699185:0:66056:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/22 (0 0) 00000080:00200000:1.0:1775764828.699191:0:66056:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x16:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1775764828.699211:0:66056:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x16:0x0](ffff8b2389a65348) for lock 0x73cfd86cfc17c0f6 00000080:00010000:1.0:1775764828.699220:0:66056:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c0f6 from it ffffa60b02357a68 00010000:00010000:1.0:1775764828.699224:0:66056:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0f6 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305299a expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.699234:0:66056:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0f6 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305299a expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764828.699256:0:66056:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0f6 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305299a expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764828.699267:0:66056:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0f6 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305299a expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:1.0:1775764828.699275:0:66056:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x16:0x0](000000000dc2588c) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0f6 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305299a expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764828.699338:0:66056:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x16:0x0]: layout version change: 4294967294 -> 0 00010000:00010000:1.0:1775764828.699359:0:66056:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0f6 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305299a expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764828.699367:0:66056:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0f6 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305299a expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764828.699572:0:66056:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x16:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764828.699585:0:66056:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764828.713174:0:66056:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x16:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1775764828.713199:0:66056:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file . :VOLATILE:0000:672BE8A9:fd=09:[0x200003ab4:0x16:0x0](ffff8b2389a65348) current size 0, valid attrs 1846, mode 8b23, uid 0, gid 0, new size 0, atime 1775764828.000000000, mtime 1775764828.000000000, ctime 1775764828.000000000, rc 0 00000080:00200008:2.0:1775764828.713261:0:66056:0:(file.c:447:ll_file_release()) START file 0x200003ab4:0x12:0x0:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 1504000 00000002:00100000:2.0:1775764828.713318:0:66056:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864f0380 x1862022098672896/t34359739298(34359739298) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764844 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764828.713330:0:66056:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764828.713335:0:66056:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4000 x1862022098674560/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764828.713358:0:66056:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098674560:192.168.202.128@tcp:35:lhsmtool_posix.0 00000100:00100000:2.0:1775764828.713435:0:66056:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764828.717156:0:66056:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098674560:192.168.202.128@tcp:35:lhsmtool_posix.0 00000002:00100000:2.0:1775764828.717191:0:66056:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864f0380 x1862022098672896/t34359739298(34359739298) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764844 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764828.717246:0:66056:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=400866d6 arg=7fb4bc000b78 00800000:00000080:2.0:1775764828.717257:0:66056:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b02357cf0 uarg=0000000000000000 00000002:00000080:2.0:1775764828.717263:0:66056:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b02357cf0 uarg=0000000000000000 00000020:00100000:2.0:1775764828.717285:0:66056:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 57, max 7 00000100:00100000:2.0:1775764828.717291:0:66056:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4000 x1862022098674816/t0(0) o57->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764828.717311:0:66056:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098674816:192.168.202.128@tcp:57:lhsmtool_posix.0 00000100:00100000:2.0:1775764828.717385:0:66056:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764828.725230:0:66056:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098674816:192.168.202.128@tcp:57:lhsmtool_posix.0 00000080:00200008:2.0:1775764828.725429:0:66056:0:(file.c:6226:ll_getattr_dentry()) START file . :VOLATILE:0000:672BE8A9:fd=09:[0x200003ab4:0x16:0x0](ffff8b2389a65348), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764828.725437:0:66056:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x16:0x0](ffff8b2389a65348),name=. :VOLATILE:0000:672BE8A9:fd=09 00000002:00010000:2.0:1775764828.725502:0:66056:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x16:0x0]) in obj [0x200003ab4:0x16:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764828.725520:0:66056:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 30 res 8589949620/22 (0 0) 00010000:00010000:2.0:1775764828.725570:0:66056:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c0fd lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 66056 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.725580:0:66056:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c0fd lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66056 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.725592:0:66056:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c0fd lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66056 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764828.725613:0:66056:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098674944:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764828.725710:0:66056:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942480 going to sleep for 11 seconds 00000080:00200000:2.0:1775764828.734321:0:66056:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x16:0x0] updating i_size 0 i_blocks 1 00000020:00010000:2.0:1775764828.734329:0:66056:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2389ea86d8 (ffff8b238a3ff008/0) at cl_lock_release():245 00000020:00010000:2.0:1775764828.734332:0:66056:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389ea86d8 (ffff8b238a3ff008/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764828.734338:0:66056:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2391066cb0 (ffff8b238a3ff5a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764828.734343:0:66056:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389ea86d8 (ffff8b238a3ff008/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764828.734346:0:66056:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2391066cb0 (ffff8b238a3ff008/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764828.734397:0:66056:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x16:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:2.0:1775764828.734708:0:66056:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=401866d8 arg=7fb4c3948910 00800000:00000080:2.0:1775764828.734721:0:66056:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b02357cf0 uarg=0000000000000000 00000002:00000080:2.0:1775764828.734731:0:66056:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b02357cf0 uarg=0000000000000000 00000020:00100000:2.0:1775764828.734764:0:66056:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 57, max 7 00000100:00100000:2.0:1775764828.734773:0:66056:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7800 x1862022098675328/t0(0) o57->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764828.734797:0:66056:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098675328:192.168.202.128@tcp:57:lhsmtool_posix.0 00000100:00100000:2.0:1775764828.734885:0:66056:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23869426c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764828.738563:0:66056:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098675328:192.168.202.128@tcp:57:lhsmtool_posix.0 00000080:00200008:2.0:1775764828.738795:0:66056:0:(file.c:2612:do_file_write_iter()) START file . :VOLATILE:0000:672BE8A9:fd=09:[0x200003ab4:0x16:0x0], ppos: 0, count: 16 00000080:00200000:2.0:1775764828.738811:0:66056:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 16 bytes 00000080:00200000:2.0:1775764828.738816:0:66056:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 16 00000080:00200000:2.0:1775764828.738849:0:66056:0:(file.c:2058:ll_file_io_generic()) . :VOLATILE:0000:672BE8A9:fd=09: write ppos: 0, bytes: 16 00000080:00200000:2.0:1775764828.738854:0:66056:0:(file.c:1815:ll_io_set_mirror()) . :VOLATILE:0000:672BE8A9:fd=09: desiginated mirror: 0 00000020:00200000:2.0:1775764828.738862:0:66056:0:(cl_io.c:214:cl_io_rw_init()) header@00000000386a8867[0x110, 1, [0x200003ab4:0x16:0x0] hash] 00000020:00200000:2.0:1775764828.738865:0:66056:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 16) 0 0 00000080:00200000:2.0:1775764828.738869:0:66056:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x16:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764828.738884:0:66056:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:2.0:1775764828.738890:0:66056:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 16) 16, 16 00020000:00200000:2.0:1775764828.738894:0:66056:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764828.738922:0:66056:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1775764828.738951:0:66056:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764828.738956:0:66056:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c81b0, osc lock ffff8b238a3f8000, flags 0x0 00010000:00010000:2.0:1775764828.738972:0:66056:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2386938000 type 11 mode 2 res 10737419264/284 (0 4095) 00010000:00010000:2.0:1775764828.739241:0:66056:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.739259:0:66056:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.739270:0:66056:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764828.739291:0:66056:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098675456:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764828.740291:0:66056:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23869426c0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764828.743423:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119072256 00000100:00100000:1.0:1775764828.743433:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119072256 00000100:00100000:1.0:1775764828.743607:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 272 00000100:00100000:1.0:1775764828.743617:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dafb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42945:x1862022119072256:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764828.743633:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17c104 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e030529af expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764828.743719:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764828.743727:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dafb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42945:x1862022119072256:12345-192.168.202.128@tcp:104: Request processed in 112us (389us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764828.743734:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 272 00010000:00010000:2.0:1775764828.743792:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764828.743799:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17c104 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e030529af expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764828.743808:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5ec00 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:2.0:1775764828.743814:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17c104 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x57b8ff0e030529af expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764828.743867:0:46150:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x16:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764828.743881:0:46150:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x16:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775764828.743952:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17c104 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e030529af expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764828.743960:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764828.743990:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17c104 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e030529af expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764828.744005:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003c855c97/0x73cfd86cfc17c104 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e030529af expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764828.744022:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764828.744025:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764828.744026:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764828.748829:0:46139:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764828.748851:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119072384 00000100:00100000:1.0:1775764828.748861:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119072384 00000100:00100000:1.0:1775764828.748915:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 273 00000100:00100000:1.0:1775764828.748925:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23b9ff2850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42309:x1862022119072384:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764828.748951:0:46139:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.748966:0:46139:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.748976:0:46139:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.748994:0:46139:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.749012:0:46139:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.749023:0:46139:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000000000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764828.749041:0:46139:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b2380031a00) 00000100:00100000:1.0:1775764828.749101:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23b9ff2850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42309:x1862022119072384:12345-192.168.202.128@tcp:105: Request processed in 177us (443us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764828.749109:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 273 00000100:00100000:2.0:1775764828.751969:0:66056:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66056:1862022098675456:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764828.752134:0:66056:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.752156:0:66056:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1775764828.752169:0:66056:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000020000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.752180:0:66056:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.752188:0:66056:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764828.752209:0:66056:0:(vvp_io.c:1286:vvp_io_write_start()) . :VOLATILE:0000:672BE8A9:fd=09: write [0, 16) append 0 append lockpos 0 00000080:00200000:2.0:1775764828.752228:0:66056:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 16 bytes 00000080:00200000:2.0:1775764828.752287:0:66056:0:(rw26.c:858:ll_write_end()) pos 0, len 16, copied 16 00000080:00200000:2.0:1775764828.752295:0:66056:0:(rw26.c:916:ll_write_end()) page@00000000789db01c[3 0000000038da6e3c 2 1 000000007f0e676c] 00000080:00200000:2.0:1775764828.752299:0:66056:0:(rw26.c:916:ll_write_end()) vmpage @00000000feb25515 17ffffc0001001 3:0 ffff8b23baeae000 0 lru 00000080:00200000:2.0:1775764828.752306:0:66056:0:(rw26.c:916:ll_write_end()) osc-page@00000000561d62ec 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 000000008819327c 000000001f1d4dcf > 3< 0 0 > 4< 0 0 8 13631488 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764828.752308:0:66056:0:(rw26.c:916:ll_write_end()) end page@00000000789db01c 00000080:00200000:2.0:1775764828.752309:0:66056:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764828.752315:0:66056:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 16 prio 0 00000008:00010000:2.0:1775764828.752336:0:66056:0:(osc_cache.c:848:osc_extent_find()) ### extent: 00000000351313f8 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764828.752357:0:66056:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a654f8, count 1, dirtied 1 00000080:00200000:2.0:1775764828.752361:0:66056:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 16 bytes, tot: 16 00000080:00200000:2.0:1775764828.752363:0:66056:0:(vvp_io.c:1353:vvp_io_write_start()) . :VOLATILE:0000:672BE8A9:fd=09: write bytes 0, result: 16 00000020:00010000:2.0:1775764828.752380:0:66056:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b5d8 (ffff8b23812ec2d8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764828.752384:0:66056:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b5d8 (ffff8b23812ec2d8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764828.752387:0:66056:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c81b0 (ffff8b23812ec518/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764828.752392:0:66056:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.752401:0:66056:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1775764828.752416:0:66056:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b5d8 (ffff8b23812ec2d8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764828.752419:0:66056:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c81b0 (ffff8b23812ec2d8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764828.752427:0:66056:0:(vvp_io.c:517:vvp_io_advance()) advancing 16 bytes 00000080:00200000:2.0:1775764828.752433:0:66056:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:2.0:1775764828.756393:0:66056:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x16:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764828.756404:0:66056:0:(file.c:2206:ll_file_io_generic()) . :VOLATILE:0000:672BE8A9:fd=09: 2 io complete with rc: 0, result: 16, restart: 0 00000080:00200000:2.0:1775764828.756421:0:66056:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x16:0x0] updating i_size 16 i_blocks 1 00000080:00200000:2.0:1775764828.756428:0:66056:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 16 00000080:00200000:2.0:1775764828.756719:0:66056:0:(file.c:5304:ll_fsync()) VFS Op:inode=[0x200003ab4:0x16:0x0](ffff8b2389a65348), start 0, end 9223372036854775807, datasync 0 00000080:00200000:2.0:1775764828.756755:0:66056:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x16:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764828.756766:0:66056:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:2.0:1775764828.756802:0:66056:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 00000000351313f8 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1775764828.756821:0:66056:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000351313f8 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764828.756870:0:66056:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00100000:2.0:1775764828.756945:0:66056:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea7800 - object 0x280000400:284 offset 0<>16 00000100:00100000:2.0:1775764828.757590:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098675712:192.168.202.128@tcp:4:lhsmtool_posix.0 00000008:00010000:2.0:1775764828.761254:0:2378:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 00000000351313f8 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:2.0:1775764828.761347:0:2378:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000351313f8 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1775764828.761372:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098675712:192.168.202.128@tcp:4:lhsmtool_posix.0 00000100:00100000:2.0:1775764828.804252:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098675968:192.168.202.128@tcp:16:lhsmtool_posix.0 00000100:00100000:2.0:1775764828.843972:0:2379:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b2389398000: committing for last_committed 34359738457 gen 1 00000100:00100000:2.0:1775764828.844043:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098675968:192.168.202.128@tcp:16:lhsmtool_posix.0 00000100:00100000:2.0:1775764828.862544:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764828.862564:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098676352:192.168.202.128@tcp:2:lhsmtool_posix.0 00000100:00100000:2.0:1775764828.873418:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098676352:192.168.202.128@tcp:2:lhsmtool_posix.0 00000100:00100000:1.0:1775764828.877243:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:2.0:1775764828.878233:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764828.878249:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098676736:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764828.884509:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098676864:192.168.202.128@tcp:1:lhsmtool_posix.0 00000100:00100000:2.0:1775764828.884615:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098676736:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764828.888593:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764828.891094:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764828.891102:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800420000020000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764828.891113:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2380031a00 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:2.0:1775764828.891119:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800428400020000 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764828.891176:0:46150:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x16:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764828.891237:0:46150:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x16:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775764828.891317:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 2/0,0 mode: --/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764828.891338:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764828.891377:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 1/0,0 mode: --/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764828.891392:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c10b lrc: 0/0,0 mode: --/PW res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e030529b6 expref: -99 pid: 66056 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764828.891417:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764828.891419:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764828.891420:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764828.891434:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764828.891448:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098676992:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764828.899850:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098676992:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764828.899884:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098676864:192.168.202.128@tcp:1:lhsmtool_posix.0 00000100:00100000:1.0:1775764828.905862:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119073024 00000100:00100000:1.0:1775764828.905873:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119073024 00000100:00100000:1.0:1775764828.905941:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 276 00000100:00100000:1.0:1775764828.905952:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dac700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42326:x1862022119073024:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764828.905969:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c0f6 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x16:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0305299a expref: -99 pid: 66056 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764828.906055:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764828.906062:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dac700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42326:x1862022119073024:12345-192.168.202.128@tcp:104: Request processed in 112us (499us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764828.906069:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 276 00000100:00100000:2.0:1775764828.917918:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098677376:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764828.922289:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764828.923373:0:66055:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.923385:0:66055:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764828.923404:0:66055:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea6a00 x1862022098672384/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/344 e 0 to 0 dl 1775764844 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764828.923415:0:66055:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.923428:0:66055:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/18 (0 0) 00000080:00200000:2.0:1775764828.923435:0:66055:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x12:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764828.923478:0:66055:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c0d3 00000080:00010000:2.0:1775764828.923487:0:66055:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c0d3 from it ffffa60b02377c18 00010000:00010000:2.0:1775764828.923491:0:66055:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.923500:0:66055:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764828.923524:0:66055:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764828.923535:0:66055:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764828.923543:0:66055:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x12:0x0](00000000580307f2) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:2.0:1775764828.923592:0:66055:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66055:1862022098677504:192.168.202.128@tcp:49:lfs.0 00000100:00100000:2.0:1775764828.923683:0:66055:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23869426c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764828.926941:0:66055:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66055:1862022098677504:192.168.202.128@tcp:49:lfs.0 00010000:00010000:2.0:1775764828.927291:0:66055:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764828.927304:0:66055:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764828.927350:0:66055:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764828.927359:0:66055:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764828.927395:0:66055:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764828.927408:0:66055:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764828.927416:0:66055:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x12:0x0](00000000580307f2) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764828.927502:0:66055:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764828.927506:0:66055:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764828.927555:0:66055:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x12:0x0]: layout version change: 4294967294 -> 4 00010000:00010000:2.0:1775764828.927564:0:66055:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764828.927573:0:66055:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00800000:00000080:2.0:1775764828.927584:0:66055:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d9 len=56 karg=ffff8b238872e980 uarg=0000000000000000 00000002:00000080:2.0:1775764828.927592:0:66055:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d9 len=56 karg=ffff8b238872e980 uarg=0000000000000000 00000020:00100000:2.0:1775764828.927617:0:66055:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 2 is allocated opc 58, max 7 00000100:00100000:2.0:1775764828.927624:0:66055:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6a00 x1862022098677760/t0(0) o58->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764828.927664:0:66055:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66055:1862022098677760:192.168.202.128@tcp:58:lfs.0 00000100:00100000:2.0:1775764828.928904:0:66055:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764828.932950:0:66055:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66055:1862022098677760:192.168.202.128@tcp:58:lfs.0 00000080:00200008:2.0:1775764828.933025:0:66055:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 154002 00000080:00200000:2.0:1775764828.971589:0:66058:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764828.971619:0:66058:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764828.972145:0:66058:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66058 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.972158:0:66058:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66058 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.972170:0:66058:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66058 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764828.972195:0:66058:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66058:1862022098678144:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764828.972302:0:66058:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764828.977961:0:66058:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66058:1862022098678144:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764828.977999:0:66058:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.978019:0:66058:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.978031:0:66058:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.978046:0:66058:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.978060:0:66058:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.978069:0:66058:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.978076:0:66058:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764828.978088:0:66058:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea6a00 x1862022098678144/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764844 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764828.978102:0:66058:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.978115:0:66058:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764828.978122:0:66058:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764828.978151:0:66058:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764828.978156:0:66058:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c112 00000080:00010000:2.0:1775764828.978162:0:66058:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764828.978165:0:66058:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c112 from it ffffa60b02307b98 00010000:00010000:2.0:1775764828.978168:0:66058:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.978180:0:66058:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764828.978205:0:66058:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764828.978231:0:66058:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=f26.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), flags=8 00000080:00200000:2.0:1775764828.978236:0:66058:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f26.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=getattr 00000002:00010000:2.0:1775764828.978256:0:66058:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f26.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764828.978298:0:66058:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66058 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.978306:0:66058:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66058 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.978316:0:66058:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66058 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764828.978338:0:66058:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66058:1862022098678272:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764828.978414:0:66058:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764828.984074:0:66058:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66058:1862022098678272:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764828.984106:0:66058:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.984124:0:66058:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.984132:0:66058:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.984145:0:66058:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.984156:0:66058:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.984169:0:66058:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.984177:0:66058:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.984184:0:66058:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764828.984209:0:66058:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea6a00 x1862022098678272/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/640 e 0 to 0 dl 1775764844 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764828.984220:0:66058:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764828.984232:0:66058:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.984242:0:66058:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/18 (0 0) 00000080:00200000:2.0:1775764828.984248:0:66058:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764828.984343:0:66058:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0x12:0x0](ffff8b238990db88): rc = 0 00000080:00200000:2.0:1775764828.984363:0:66058:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x12:0x0]: layout version change: 4 -> 4 00000080:00010000:2.0:1775764828.984375:0:66058:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c119 00000080:00010000:2.0:1775764828.984388:0:66058:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00010000:2.0:1775764828.984392:0:66058:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c119 from it ffffa60b02307b60 00010000:00010000:2.0:1775764828.984395:0:66058:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764828.984402:0:66058:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764829.112973:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764829.979920:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764830.024758:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764830.024767:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17c120 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03052a1f expref: -99 pid: 66070 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764830.024776:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5f000 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764830.024782:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17c120 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03052a1f expref: -99 pid: 66070 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764830.024789:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17c120 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03052a1f expref: -99 pid: 66070 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764830.024798:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17c120 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052a1f expref: -99 pid: 66070 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764830.024821:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764830.024858:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17c120 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052a1f expref: -99 pid: 66070 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764830.024866:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764830.024899:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17c120 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052a1f expref: -99 pid: 66070 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764830.024907:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f000/0x73cfd86cfc17c120 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052a1f expref: -99 pid: 66070 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764830.024923:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764830.024925:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764830.024927:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764830.024935:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764830.024952:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098679040:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764830.026999:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098679040:192.168.202.128@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764830.135669:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1775764830.932659:0:66076:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764830.932674:0:66076:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764830.932697:0:66076:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764830.932715:0:66076:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764830.932730:0:66076:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764830.932737:0:66076:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764830.932745:0:66076:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764830.932749:0:66076:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c112 00000080:00010000:2.0:1775764830.932756:0:66076:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c112 from it ffffa60b0232fcb8 00010000:00010000:2.0:1775764830.932760:0:66076:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764830.932769:0:66076:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764830.932793:0:66076:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764830.932799:0:66076:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764830.932805:0:66076:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764830.932813:0:66076:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764830.932818:0:66076:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764830.932822:0:66076:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764830.932824:0:66076:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c112 00000080:00010000:2.0:1775764830.932827:0:66076:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c112 from it ffffa60b0232fb58 00010000:00010000:2.0:1775764830.932830:0:66076:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764830.932837:0:66076:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764830.932845:0:66076:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764830.932852:0:66076:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=8 00000080:00200008:2.0:1775764830.932858:0:66076:0:(file.c:6226:ll_getattr_dentry()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764830.932862:0:66076:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88),name=f26.sanity-pcc 00000002:00010000:2.0:1775764830.932884:0:66076:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764830.932892:0:66076:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764830.932900:0:66076:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764830.932906:0:66076:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764830.932910:0:66076:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00010000:2.0:1775764830.932912:0:66076:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c119 00000080:00010000:2.0:1775764830.932916:0:66076:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c119 from it ffffa60b0232fcb8 00010000:00010000:2.0:1775764830.932918:0:66076:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764830.932924:0:66076:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764830.932981:0:66076:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1775764830.932991:0:66076:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x12:0x0] 00000008:00010000:2.0:1775764830.933259:0:66076:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764830.933263:0:66076:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8930, osc lock ffff8b238a3f8000, flags 0x400001002 00010000:00010000:2.0:1775764830.933316:0:66076:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764830.933329:0:66076:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1775764830.933924:0:66076:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1775764830.935841:0:66076:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x12:0x0] updating i_size 16 i_blocks 11 00000020:00010000:1.0:1775764830.935846:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027eda0 (ffff8b23812ec998/0) at cl_lock_release():245 00000020:00010000:1.0:1775764830.935848:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027eda0 (ffff8b23812ec998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764830.935852:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8930 (ffff8b23812ec7e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764830.935854:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027eda0 (ffff8b23812ec998/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764830.935856:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8930 (ffff8b23812ec998/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764830.935891:0:66076:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/1, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764830.935941:0:66076:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764830.935956:0:66076:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764830.935970:0:66076:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764830.935981:0:66076:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764830.935989:0:66076:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764830.935997:0:66076:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764830.936001:0:66076:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c112 00000080:00010000:1.0:1775764830.936016:0:66076:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c112 from it ffffa60b0232fbf8 00010000:00010000:1.0:1775764830.936020:0:66076:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764830.936028:0:66076:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764830.936038:0:66076:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764830.936045:0:66076:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=265 00000080:00200000:1.0:1775764830.936049:0:66076:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88), inode mode 81a4 mask 44 00000080:00200008:1.0:1775764830.936055:0:66076:0:(file.c:1057:ll_file_open()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100000 00000002:00010000:1.0:1775764830.936068:0:66076:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 04240100001 00010000:00010000:1.0:1775764830.936111:0:66076:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c12e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66076 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764830.936119:0:66076:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c12e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66076 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764830.936130:0:66076:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764830.936136:0:66076:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ff800 x1862022098679680/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764830.936144:0:66076:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c12e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66076 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764830.936159:0:66076:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66076:1862022098679680:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764830.936305:0:66076:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1c480 going to sleep for 11 seconds 00000100:00100000:1.0:1775764830.940037:0:66076:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66076:1862022098679680:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764830.940079:0:66076:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c12e lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a42 expref: -99 pid: 66076 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764830.940090:0:66076:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c12e lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a42 expref: -99 pid: 66076 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764830.940104:0:66076:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c12e lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052a42 expref: -99 pid: 66076 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764830.940119:0:66076:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c12e lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a42 expref: -99 pid: 66076 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764830.940138:0:66076:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23864ff800 x1862022098679680/t34359739314(34359739314) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764846 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764830.940156:0:66076:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864ff800 x1862022098679680/t34359739314(34359739314) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764846 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764830.940171:0:66076:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c12e lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a42 expref: -99 pid: 66076 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764830.940190:0:66076:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c12e lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a42 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764830.940349:0:66076:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/18 (0 0) 00000080:00200000:1.0:1775764830.940360:0:66076:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764830.940765:0:66076:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x12:0x0]: layout version change: 4 -> 4 00000080:00010000:1.0:1775764830.940780:0:66076:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c12e 00000080:00010000:1.0:1775764830.940791:0:66076:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c12e from it ffffa60b0232fbf0 00010000:00010000:1.0:1775764830.940857:0:66076:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c12e lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052a42 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764830.940873:0:66076:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c12e lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052a42 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764830.940905:0:66076:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864ff800 x1862022098679680/t34359739314(34359739314) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764846 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764830.940992:0:66076:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88) cmd=400866fc arg=7ffcbadde768 00000080:00010000:1.0:1775764830.941029:0:66076:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000002:00010000:1.0:1775764830.941038:0:66076:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200003ab4:0x12:0x0], intent: layout flags 02 00010000:00010000:1.0:1775764830.941055:0:66076:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764830.943234:0:66076:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x12:0x0]: losing layout lock 00010000:00010000:1.0:1775764830.943754:0:66076:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c12e lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052a42 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764830.943786:0:66076:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x12:0x0]: losing layout lock 00010000:00010000:1.0:1775764830.943803:0:66076:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764830.943824:0:66076:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x12:0x0]: losing layout lock 00010000:00010000:1.0:1775764830.943867:0:66076:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764830.943875:0:66076:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c12e lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052a42 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764830.943882:0:66076:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764830.943888:0:66076:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:1.0:1775764830.943891:0:66076:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c119 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052a11 expref: -99 pid: 66058 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764830.943903:0:66076:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c12e lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052a42 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764830.943912:0:66076:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c0d3 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052923 expref: -99 pid: 66055 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764830.943943:0:66076:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x12:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66076 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764830.943950:0:66076:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66076 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764830.943971:0:66076:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764830.943984:0:66076:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fe300 x1862022098679936/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 392/880 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764830.943993:0:66076:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66076 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764830.944019:0:66076:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66076:1862022098679936:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764830.944833:0:66076:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1c480 going to sleep for 11 seconds 00000100:00100000:1.0:1775764830.949261:0:66076:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66076:1862022098679936:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764830.949289:0:66076:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 4/0,1 mode: --/EX res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a57 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764830.949301:0:66076:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 4/0,1 mode: --/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a57 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764830.949312:0:66076:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052a57 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764830.949324:0:66076:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a57 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764830.949336:0:66076:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052a57 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764830.949344:0:66076:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052a57 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764830.949351:0:66076:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052a57 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764830.949367:0:66076:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23864fe300 x1862022098679936/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 392/400 e 0 to 0 dl 1775764846 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764830.949378:0:66076:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a57 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764830.949391:0:66076:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a57 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764830.949403:0:66076:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/18 (0 0) 00000080:00200000:1.0:1775764830.949409:0:66076:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x12:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1775764830.949427:0:66076:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c135 00000080:00010000:1.0:1775764830.949435:0:66076:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c135 from it ffffa60b0232fc10 00010000:00010000:1.0:1775764830.949438:0:66076:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052a57 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764830.949446:0:66076:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a57 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764830.951788:0:66076:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a57 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764830.951803:0:66076:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a57 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:1.0:1775764830.951812:0:66076:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x12:0x0](00000000580307f2) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a57 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764830.951896:0:66076:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x12:0x0]: layout version change: 4294967294 -> 4 00010000:00010000:1.0:1775764830.951918:0:66076:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052a57 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764830.951927:0:66076:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052a57 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:1.0:1775764830.952758:0:66076:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 0, count: 1048576 00000080:00200000:1.0:1775764830.952787:0:66076:0:(rw.c:1928:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1775764830.952816:0:66076:0:(file.c:2058:ll_file_io_generic()) f26.sanity-pcc: read ppos: 0, bytes: 1048576 00000080:00200000:1.0:1775764830.952821:0:66076:0:(file.c:1815:ll_io_set_mirror()) f26.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764830.952828:0:66076:0:(cl_io.c:214:cl_io_rw_init()) header@000000002eb11a92[0x110, 1, [0x200003ab4:0x12:0x0] hash] 00000020:00200000:1.0:1775764830.952830:0:66076:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 1048576) 0 0 00000080:00200000:1.0:1775764830.952835:0:66076:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764830.952854:0:66076:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:1.0:1775764830.952858:0:66076:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764830.952889:0:66076:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:1.0:1775764830.952904:0:66076:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764830.952909:0:66076:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c58b0, osc lock ffff8b2382cf6e40, flags 0x0 00010000:00010000:1.0:1775764830.952931:0:66076:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764830.953230:0:66076:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764830.953244:0:66076:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764830.953257:0:66076:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764830.953264:0:66076:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764830.953275:0:66076:0:(vvp_io.c:826:vvp_io_read_start()) f26.sanity-pcc: read [0, 1048576) 00000001:00010000:1.0:1775764830.953291:0:66076:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x12:0x0] 00000008:00010000:1.0:1775764830.953298:0:66076:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764830.953300:0:66076:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5f30, osc lock ffff8b2382cf7a20, flags 0x400001002 00010000:00010000:1.0:1775764830.953308:0:66076:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 4/2,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764830.953317:0:66076:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 4/2,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1775764830.953326:0:66076:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764830.953330:0:66076:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 7/2,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764830.953337:0:66076:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 6/1,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1775764830.953343:0:66076:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764830.953350:0:66076:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x12:0x0] updating i_size 16 i_blocks 11 00000020:00010000:1.0:1775764830.953355:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f18000 (ffff8b2387cba518/0) at cl_lock_release():245 00000020:00010000:1.0:1775764830.953358:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f18000 (ffff8b2387cba518/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764830.953360:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5f30 (ffff8b2387cba878/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764830.953364:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f18000 (ffff8b2387cba518/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764830.953366:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5f30 (ffff8b2387cba518/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764830.953376:0:66076:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:0 00000008:00010000:1.0:1775764830.953415:0:66076:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b238a0ce480 00000008:00010000:1.0:1775764830.954889:0:54744:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b238a0ce480 00010000:00010000:1.0:1775764830.954930:0:54744:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00100000:1.0:1775764830.954994:0:54744:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23864fe300 - object 0x280000400:284 offset 0<>4096 00000100:00100000:1.0:1775764830.955130:0:2376:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764830.955152:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098680064:192.168.202.128@tcp:3:lfs.0 00000008:00010000:1.0:1775764830.960129:0:2376:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8b238a0ce480 00000020:00200000:1.0:1775764830.960155:0:2376:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1775764830.960175:0:2376:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8b238a0ce480 00000100:00100000:1.0:1775764830.960202:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098680064:192.168.202.128@tcp:3:lfs.0 00000020:00010000:2.0:1775764830.960250:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb75d8 (ffff8b2387cba518/0) at cl_lock_release():245 00000020:00010000:2.0:1775764830.960256:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb75d8 (ffff8b2387cba518/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764830.960260:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c58b0 (ffff8b2387cba878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764830.960270:0:66076:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764830.960286:0:66076:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00010000:2.0:1775764830.960303:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb75d8 (ffff8b2387cba518/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764830.960306:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c58b0 (ffff8b2387cba518/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764830.960316:0:66076:0:(vvp_io.c:517:vvp_io_advance()) advancing 16 bytes 00000080:00200000:2.0:1775764830.960342:0:66076:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764830.960347:0:66076:0:(file.c:2206:ll_file_io_generic()) f26.sanity-pcc: 1 io complete with rc: 0, result: 16, restart: 0 00000080:00200000:2.0:1775764830.960353:0:66076:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 16 00000080:00200008:2.0:1775764830.960527:0:66076:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0], ppos: 16, count: 1048576 00000080:00200000:2.0:1775764830.960552:0:66076:0:(file.c:2058:ll_file_io_generic()) f26.sanity-pcc: read ppos: 16, bytes: 1048576 00000080:00200000:2.0:1775764830.960557:0:66076:0:(file.c:1815:ll_io_set_mirror()) f26.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764830.960563:0:66076:0:(cl_io.c:214:cl_io_rw_init()) header@000000002eb11a92[0x110, 2, [0x200003ab4:0x12:0x0] hash] 00000020:00200000:2.0:1775764830.960565:0:66076:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [16, 1048592) 0 0 00000080:00200000:2.0:1775764830.960569:0:66076:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764830.960587:0:66076:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 16 chunk: [16, 1048592) 1048592, 1048576 00020000:00200000:2.0:1775764830.960591:0:66076:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764830.960635:0:66076:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 256] 00000008:00010000:2.0:1775764830.960650:0:66076:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764830.960656:0:66076:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c86b0, osc lock ffff8b238a3f8be0, flags 0x0 00010000:00010000:2.0:1775764830.960678:0:66076:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764830.960694:0:66076:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1052671) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764830.960709:0:66076:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764830.960718:0:66076:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764830.960725:0:66076:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764830.960735:0:66076:0:(vvp_io.c:826:vvp_io_read_start()) f26.sanity-pcc: read [16, 1048592) 00000001:00010000:2.0:1775764830.960751:0:66076:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x12:0x0] 00000008:00010000:2.0:1775764830.960757:0:66076:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764830.960759:0:66076:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8a30, osc lock ffff8b238a3f8130, flags 0x400001002 00010000:00010000:2.0:1775764830.960766:0:66076:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 4/2,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764830.960775:0:66076:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 4/2,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1775764830.960784:0:66076:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764830.960787:0:66076:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 7/2,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764830.960800:0:66076:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 6/1,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1775764830.960807:0:66076:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764830.960812:0:66076:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x12:0x0] updating i_size 16 i_blocks 11 00000020:00010000:2.0:1775764830.960816:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027cb68 (ffff8b23812ece18/0) at cl_lock_release():245 00000020:00010000:2.0:1775764830.960819:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027cb68 (ffff8b23812ece18/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764830.960822:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8a30 (ffff8b23812ecf38/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764830.960824:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027cb68 (ffff8b23812ece18/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764830.960826:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8a30 (ffff8b23812ece18/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764830.960833:0:66076:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:16 00000020:00010000:2.0:1775764830.960839:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b7b8 (ffff8b23812ece18/0) at cl_lock_release():245 00000020:00010000:2.0:1775764830.960841:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b7b8 (ffff8b23812ece18/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764830.960844:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c86b0 (ffff8b23812ecf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764830.960848:0:66076:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764830.960856:0:66076:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000003cfe7f4b/0x73cfd86cfc17c127 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x11c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052a34 expref: -99 pid: 66076 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00010000:2.0:1775764830.960866:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b7b8 (ffff8b23812ece18/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764830.960868:0:66076:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c86b0 (ffff8b23812ece18/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764830.960872:0:66076:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1775764830.960892:0:66076:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x12:0x0] ignore/verify layout 0/0, layout version 4 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764830.960896:0:66076:0:(file.c:2206:ll_file_io_generic()) f26.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1775764830.960899:0:66076:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:2.0:1775764830.961577:0:66076:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100000 00000002:00100000:2.0:1775764830.961823:0:66076:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864ff800 x1862022098679680/t34359739314(34359739314) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764846 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764830.961834:0:66076:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764830.961838:0:66076:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7800 x1862022098680320/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764830.961859:0:66076:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66076:1862022098680320:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764830.961940:0:66076:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764830.987868:0:66076:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66076:1862022098680320:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764830.987897:0:66076:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864ff800 x1862022098679680/t34359739314(34359739314) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764846 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764831.149907:0:66093:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764831.149934:0:66093:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764831.149952:0:66093:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764831.149970:0:66093:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764831.149978:0:66093:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764831.149987:0:66093:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764831.149991:0:66093:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c112 00000080:00010000:2.0:1775764831.149998:0:66093:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c112 from it ffffa60b023cfbf8 00010000:00010000:2.0:1775764831.150002:0:66093:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764831.150114:0:66093:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c112 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030529f5 expref: -99 pid: 66058 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764831.150130:0:66093:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764831.150146:0:66093:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f26.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b23a0dc5680, open_flags 8000, mode 0 opened 0 00000080:00200000:2.0:1775764831.150154:0:66093:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f26.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open 00000002:00010000:2.0:1775764831.150168:0:66093:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f26.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100001 00010000:00010000:2.0:1775764831.150221:0:66093:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66093 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764831.150229:0:66093:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66093 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764831.150242:0:66093:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764831.150255:0:66093:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7800 x1862022098680704/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764831.150264:0:66093:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66093 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764831.150287:0:66093:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:66093:1862022098680704:192.168.202.128@tcp:101:cat.0 00000100:00100000:2.0:1775764831.150639:0:66093:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764831.158555:0:66093:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:66093:1862022098680704:192.168.202.128@tcp:101:cat.0 00010000:00010000:2.0:1775764831.158587:0:66093:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764831.158604:0:66093:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764831.158622:0:66093:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764831.158633:0:66093:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764831.158645:0:66093:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764831.158659:0:66093:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764831.158667:0:66093:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764831.158674:0:66093:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764831.158686:0:66093:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23b9ea7800 x1862022098680704/t34359739316(34359739316) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764847 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764831.158698:0:66093:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7800 x1862022098680704/t34359739316(34359739316) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764847 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764831.158711:0:66093:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764831.158724:0:66093:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764831.158736:0:66093:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/18 (0 0) 00000080:00200000:2.0:1775764831.158742:0:66093:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764831.158843:0:66093:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0x12:0x0](ffff8b238990db88): rc = 0 00000080:00200000:2.0:1775764831.158862:0:66093:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x12:0x0]: layout version change: 4 -> 4 00000080:00010000:2.0:1775764831.158871:0:66093:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c13c 00000080:00010000:2.0:1775764831.158885:0:66093:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00200008:2.0:1775764831.158898:0:66093:0:(file.c:1057:ll_file_open()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 100000 00000002:00100000:2.0:1775764831.158918:0:66093:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7800 x1862022098680704/t34359739316(34359739316) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764847 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764831.158974:0:66093:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c13c from it ffff8b238504cae0 00010000:00010000:2.0:1775764831.158978:0:66093:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764831.158988:0:66093:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764831.159258:0:66093:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88), inode mode 81a4 mask 44 00000080:00200008:2.0:1775764831.159289:0:66093:0:(file.c:6226:ll_getattr_dentry()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764831.159295:0:66093:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88),name=f26.sanity-pcc 00000002:00010000:2.0:1775764831.159314:0:66093:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x12:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764831.159324:0:66093:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764831.159336:0:66093:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764831.159348:0:66093:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764831.159352:0:66093:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) 00000080:00010000:2.0:1775764831.159355:0:66093:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c13c 00000080:00010000:2.0:1775764831.159358:0:66093:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c13c from it ffffa60b023cfcd8 00010000:00010000:2.0:1775764831.159360:0:66093:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764831.159367:0:66093:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.159860:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764831.164204:0:66093:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:66093:1862022098680960:192.168.202.128@tcp:35:cat.0 00000002:00100000:1.0:1775764831.164236:0:66093:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea7800 x1862022098680704/t34359739316(34359739316) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764847 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cat.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764831.384815:0:66028:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:66028:1862022098681216:192.168.202.128@tcp:35:multiop.0 00000002:00100000:1.0:1775764831.384850:0:66028:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fc000 x1862022098671104/t34359739297(34359739297) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764841 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764831.548366:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119074304 00000100:00100000:1.0:1775764831.548378:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119074304 00000100:00100000:1.0:1775764831.548442:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 279 00000100:00100000:1.0:1775764831.548486:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dae680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119074304:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764831.548517:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764831.548596:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764831.548602:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dae680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119074304:12345-192.168.202.128@tcp:104: Request processed in 152us (340us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764831.548622:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 279 00010000:00010000:2.0:1775764831.548814:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764831.548821:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764831.548830:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2380030a00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764831.548834:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764831.548843:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764831.548852:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764831.548881:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c135 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x12:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052a57 expref: -99 pid: 66076 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764831.548926:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764831.548933:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764831.548973:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764831.548980:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c13c lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x12:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052a88 expref: -99 pid: 66093 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764831.548996:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764831.548997:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764831.548999:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00010000:1.0:1775764831.555925:0:66133:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x12:0x0](ffff8b238990db88) for lock 0x73cfd86cfc17c143 00000080:00010000:1.0:1775764831.555934:0:66133:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c143 from it ffffa60b0241fbf0 00010000:00010000:1.0:1775764831.555937:0:66133:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c143 lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052a9d expref: -99 pid: 66133 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764831.555945:0:66133:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c143 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a9d expref: -99 pid: 66133 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764831.555982:0:66133:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382dad500 x1862022098681472/t34359739319(34359739319) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764847 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764831.556082:0:66133:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x12:0x0](ffff8b238990db88) cmd=c00466fc arg=7ffdc8c3b90c 00000080:00200008:1.0:1775764831.556145:0:66133:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x12:0x0](ffff8b238990db88), flags 154002 00010000:00010000:1.0:1775764831.556162:0:66133:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c143 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052a9d expref: -99 pid: 66133 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00100000:1.0:1775764831.576964:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764831.577901:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764831.577908:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c143 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03052a9d expref: -99 pid: 66133 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.577917:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2381497200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764831.577923:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c143 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03052a9d expref: -99 pid: 66133 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.577931:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c143 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03052a9d expref: -99 pid: 66133 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.577938:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c143 lrc: 3/0,0 mode: CW/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x57b8ff0e03052a9d expref: -99 pid: 66133 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000002:00100000:2.0:1775764831.578012:0:46149:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b2382dad500 x1862022098681472/t34359739319(34359739319) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764847 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764831.578023:0:46149:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764831.578028:0:46149:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6a00 x1862022098681984/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764831.578052:0:46149:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:46149:1862022098681984:192.168.202.128@tcp:35:ldlm_bl.0 00000100:00100000:2.0:1775764831.578104:0:46149:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764831.582166:0:46149:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:46149:1862022098681984:192.168.202.128@tcp:35:ldlm_bl.0 00000002:00100000:2.0:1775764831.582202:0:46149:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b2382dad500 x1862022098681472/t34359739319(34359739319) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764847 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764831.582282:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c143 lrc: 2/0,0 mode: --/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03052a9d expref: -99 pid: 66133 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.582292:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764831.582321:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c143 lrc: 1/0,0 mode: --/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03052a9d expref: -99 pid: 66133 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.582329:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c143 lrc: 0/0,0 mode: --/CW res: [0x200003ab4:0x12:0x0].0x0 bits 0x5/0x5 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03052a9d expref: -99 pid: 66133 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.582346:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764831.582347:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764831.582349:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1775764831.598537:0:65417:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764831.598557:0:65417:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764831.598627:0:65417:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17c151 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65417 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764831.598637:0:65417:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17c151 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65417 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764831.598648:0:65417:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741600/0x73cfd86cfc17c151 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65417 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764831.598672:0:65417:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:65417:1862022098682496:192.168.202.128@tcp:101:bash.0 00000100:00100000:1.0:1775764831.598757:0:65417:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1cc00 going to sleep for 11 seconds 00000080:00200008:1.0:1775764831.682336:0:66140:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764831.682350:0:66140:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764831.682373:0:66140:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764831.682440:0:66140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66140 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764831.682673:0:66140:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66140 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764831.682687:0:66140:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66140 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764831.682716:0:66140:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66140:1862022098683264:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764831.682877:0:66140:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1cc00 going to sleep for 11 seconds 00000100:00100000:1.0:1775764831.694729:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764831.698647:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764831.698663:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c158 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03052b06 expref: -99 pid: 65417 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.698677:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2381497600 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764831.698683:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c158 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052b06 expref: -99 pid: 65417 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.698692:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c158 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052b06 expref: -99 pid: 65417 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.698702:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c158 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03052b06 expref: -99 pid: 65417 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764831.698768:0:46150:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x17:0x0]: losing layout lock 00010000:00010000:2.0:1775764831.698845:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c158 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052b06 expref: -99 pid: 65417 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.698852:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764831.698878:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c158 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052b06 expref: -99 pid: 65417 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.698885:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c158 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052b06 expref: -99 pid: 65417 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.698905:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764831.698907:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764831.698909:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764831.698940:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098683648:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764831.713410:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098683648:192.168.202.128@tcp:103:ldlm_bl.0 00000002:00100000:2.0:1775764831.714850:0:66140:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864f0a80 x1862022098683904/t34359739325(34359739325) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764847 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764831.714864:0:66140:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x17:0x0](ffff8b23899c63c8) for lock 0x73cfd86cfc17c174 00000080:00010000:2.0:1775764831.714871:0:66140:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c174 from it ffffa60b023bfcc0 00010000:00010000:2.0:1775764831.714874:0:66140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c174 lrc: 3/0,1 mode: PW/PW res: [0x200003ab4:0x17:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e03052b45 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764831.714880:0:66140:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c174 lrc: 2/0,0 mode: PW/PW res: [0x200003ab4:0x17:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e03052b45 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200080:2.0:1775764831.715225:0:66140:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x17:0x0](ffff8b23899c63c8) cmd=c00c66f3 arg=19de2a0 00000080:00200008:2.0:1775764831.716275:0:66140:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x17:0x0], ppos: 0, count: 1048576 00000080:00200000:2.0:1775764831.716316:0:66140:0:(file.c:2058:ll_file_io_generic()) f26.sanity-pcc: read ppos: 0, bytes: 1048576 00000080:00200000:2.0:1775764831.716322:0:66140:0:(file.c:1815:ll_io_set_mirror()) f26.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764831.716330:0:66140:0:(cl_io.c:214:cl_io_rw_init()) header@000000002e454a90[0x110, 2, [0x200003ab4:0x17:0x0] hash] 00000020:00200000:2.0:1775764831.716333:0:66140:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 1048576) 2048 0 00000080:00200000:2.0:1775764831.716338:0:66140:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x17:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00010000:00010000:2.0:1775764831.716359:0:66140:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 31 res 8589949620/23 (0 0) 00000080:00010000:2.0:1775764831.717603:0:66140:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x17:0x0](ffff8b23899c63c8) 00000002:00010000:2.0:1775764831.717631:0:66140:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x17:0x0]) in obj [0x200003ab4:0x17:0x0], intent: layout flags 0 00010000:00010000:2.0:1775764831.717702:0:66140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66140 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764831.717712:0:66140:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66140 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764831.717722:0:66140:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66140 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764831.717749:0:66140:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66140:1862022098684032:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764831.717910:0:66140:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764831.721230:0:66140:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66140:1862022098684032:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764831.721266:0:66140:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052b53 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764831.721282:0:66140:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b53 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764831.721296:0:66140:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea7100 x1862022098684032/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/400 e 0 to 0 dl 1775764847 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764831.721305:0:66140:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b53 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764831.721317:0:66140:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b53 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764831.721330:0:66140:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/23 (0 0) 00000080:00200000:2.0:1775764831.721336:0:66140:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x17:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764831.721353:0:66140:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x17:0x0](ffff8b23899c63c8) for lock 0x73cfd86cfc17c17b 00000080:00010000:2.0:1775764831.721361:0:66140:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c17b from it ffffa60b023bf778 00010000:00010000:2.0:1775764831.721365:0:66140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052b53 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764831.721374:0:66140:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b53 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764831.721395:0:66140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b53 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764831.721407:0:66140:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b53 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764831.721415:0:66140:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x17:0x0](00000000e555a973) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b53 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1775764831.723226:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098684160:192.168.202.128@tcp:1:lfs.0 00000100:00100000:1.0:1775764831.726700:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764831.728428:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764831.728438:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b9df339d/0x73cfd86cfc17c15f lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e03052b14 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764831.728463:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2381496000 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:2.0:1775764831.728470:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b9df339d/0x73cfd86cfc17c15f lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800428400000000 nid: local remote: 0x57b8ff0e03052b14 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00010000:2.0:1775764831.728605:0:46149:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 000000000bb64da8 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b9df339d/0x73cfd86cfc17c15f lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800429400000000 nid: local remote: 0x57b8ff0e03052b14 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00010000:2.0:1775764831.728630:0:46149:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 000000000bb64da8 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b9df339d/0x73cfd86cfc17c15f lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800429400000000 nid: local remote: 0x57b8ff0e03052b14 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764831.728681:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b9df339d/0x73cfd86cfc17c15f lrc: 5/0,0 mode: PW/PW res: [0x240000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800429400000000 nid: local remote: 0x57b8ff0e03052b14 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00100000:2.0:1775764831.728757:0:46149:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea7100 - object 0x240000400:285 offset 0<>16 00000008:00010000:2.0:1775764831.732476:0:2378:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 000000000bb64da8 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b9df339d/0x73cfd86cfc17c15f lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800429400000000 nid: local remote: 0x57b8ff0e03052b14 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00010000:2.0:1775764831.732655:0:2378:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000bb64da8 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b9df339d/0x73cfd86cfc17c15f lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800429400000000 nid: local remote: 0x57b8ff0e03052b14 expref: -99 pid: 65417 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000100:00100000:2.0:1775764831.732685:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098684288:192.168.202.128@tcp:4:lfs.0 00000100:00100000:2.0:1775764831.733300:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098684416:192.168.202.128@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764831.738417:0:66140:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c174 lrc: 2/0,0 mode: PW/PW res: [0x200003ab4:0x17:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x57b8ff0e03052b45 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764831.738437:0:66140:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c174 lrc: 0/0,0 mode: --/PW res: [0x200003ab4:0x17:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x57b8ff0e03052b45 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764831.738568:0:66140:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864f0a80 x1862022098683904/t34359739325(34359739325) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764847 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764831.738580:0:66140:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764831.738586:0:66140:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd880 x1862022098684672/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 504/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764831.738617:0:66140:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66140:1862022098684672:192.168.202.128@tcp:35:lfs.0 00000100:00100000:1.0:1775764831.738728:0:66140:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1c480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764831.739789:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098684416:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764831.744832:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764831.745908:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764831.745915:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03052b53 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.745924:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2380030a00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764831.745929:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052b53 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.745938:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052b53 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.745946:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 3/0,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03052b53 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764831.745994:0:46150:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x17:0x0]: losing layout lock 00010000:00010000:2.0:1775764831.746061:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 2/0,0 mode: --/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052b53 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.746075:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764831.746103:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052b53 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.746111:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c17b lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052b53 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764831.746136:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764831.746138:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764831.746139:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764831.746148:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764831.746163:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098684800:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764831.756007:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098684800:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764831.757386:0:66140:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66140:1862022098684672:192.168.202.128@tcp:35:lfs.0 00000002:00100000:1.0:1775764831.757446:0:66140:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864f0a80 x1862022098683904/t34359739325(34359739325) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764847 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1775764831.757531:0:66140:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x17:0x0](ffff8b23899c63c8) 00000002:00010000:1.0:1775764831.757542:0:66140:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x17:0x0]) in obj [0x200003ab4:0x17:0x0], intent: layout flags 0 00010000:00010000:1.0:1775764831.757596:0:66140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66140 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764831.757622:0:66140:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66140 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764831.757634:0:66140:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66140 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764831.757654:0:66140:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66140:1862022098684928:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764831.757721:0:66140:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1c480 going to sleep for 11 seconds 00000100:00100000:1.0:1775764831.762214:0:66140:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66140:1862022098684928:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764831.762246:0:66140:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052b68 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764831.762262:0:66140:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b68 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764831.762276:0:66140:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23864fd880 x1862022098684928/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/400 e 0 to 0 dl 1775764847 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764831.762286:0:66140:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b68 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764831.762299:0:66140:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b68 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764831.762311:0:66140:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/23 (0 0) 00000080:00200000:1.0:1775764831.762317:0:66140:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x17:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1775764831.762333:0:66140:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x17:0x0](ffff8b23899c63c8) for lock 0x73cfd86cfc17c182 00000080:00010000:1.0:1775764831.762341:0:66140:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c182 from it ffffa60b023bfb70 00010000:00010000:1.0:1775764831.762345:0:66140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052b68 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764831.762355:0:66140:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b68 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764831.762631:0:66140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b68 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764831.762645:0:66140:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b68 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:1.0:1775764831.762653:0:66140:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x17:0x0](00000000e555a973) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b68 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764831.762729:0:66140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052b68 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764831.762736:0:66140:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b68 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764831.762762:0:66140:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x17:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764831.762774:0:66140:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764831.762817:0:66140:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x17:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1775764831.762836:0:66140:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b68 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764831.762845:0:66140:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b68 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1775764831.762851:0:66140:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x17:0x0](00000000e555a973) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b68 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764831.762885:0:66140:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x17:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764831.762890:0:66140:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764831.762916:0:66140:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x17:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764831.762964:0:66140:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x17:0x0]: layout version change: 4294967294 -> 1 00010000:00010000:1.0:1775764831.762973:0:66140:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052b68 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764831.762980:0:66140:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c182 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052b68 expref: -99 pid: 66140 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:1.0:1775764831.764053:0:66140:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x17:0x0](ffff8b23899c63c8), flags 104002 00000002:00100000:1.0:1775764831.764118:0:66140:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864f2a00 x1862022098683520/t34359739324(34359739324) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764847 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764831.764130:0:66140:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764831.764134:0:66140:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd880 x1862022098685184/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764831.764157:0:66140:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66140:1862022098685184:192.168.202.128@tcp:35:lfs.0 00000100:00100000:1.0:1775764831.764263:0:66140:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1c480 going to sleep for 11 seconds 00000100:00100000:1.0:1775764831.768016:0:66140:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66140:1862022098685184:192.168.202.128@tcp:35:lfs.0 00000002:00100000:1.0:1775764831.768051:0:66140:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864f2a00 x1862022098683520/t34359739324(34359739324) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764847 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764831.880429:0:66150:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764831.880623:0:66150:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764831.880643:0:66150:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b22 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764831.880660:0:66150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b22 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764831.880667:0:66150:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764831.880678:0:66150:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764831.880682:0:66150:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c166 00000080:00010000:1.0:1775764831.880690:0:66150:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c166 from it ffffa60b0246fbb0 00010000:00010000:1.0:1775764831.880694:0:66150:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052b22 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764831.880703:0:66150:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b22 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764831.880718:0:66150:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764831.880727:0:66150:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:1.0:1775764831.880751:0:66150:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffef36cb9f0 00000100:00100000:1.0:1775764831.880818:0:66150:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66150:1862022098685312:192.168.202.128@tcp:34:lfs.0 00000100:00100000:1.0:1775764831.880981:0:66150:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1c480 going to sleep for 11 seconds 00000100:00100000:1.0:1775764831.885423:0:66150:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66150:1862022098685312:192.168.202.128@tcp:34:lfs.0 00000080:00200008:1.0:1775764831.885538:0:66150:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000100:00100000:2.0:1775764832.317870:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2378:1862022098685568:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764832.317979:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1775764832.317987:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2378:1862022098685440:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764832.318030:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098685696:192.168.202.128@tcp:400:kworker.0 00000080:00200000:2.0:1775764832.329263:0:66166:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764832.329290:0:66166:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764832.329308:0:66166:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b22 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764832.329323:0:66166:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b22 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764832.329334:0:66166:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764832.329343:0:66166:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764832.329346:0:66166:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c166 00000080:00010000:2.0:1775764832.329353:0:66166:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c166 from it ffffa60b02467bf8 00010000:00010000:2.0:1775764832.329356:0:66166:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052b22 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764832.329365:0:66166:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b22 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764832.329380:0:66166:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764832.329393:0:66166:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764832.329399:0:66166:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764832.329404:0:66166:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b22 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764832.329412:0:66166:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b22 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764832.329418:0:66166:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764832.329421:0:66166:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764832.329423:0:66166:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c166 00000080:00010000:2.0:1775764832.329427:0:66166:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c166 from it ffffa60b02467b38 00010000:00010000:2.0:1775764832.329429:0:66166:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052b22 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764832.329440:0:66166:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052b22 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764832.329446:0:66166:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:2.0:1775764832.329479:0:66166:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f26.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b23903e9b00, open_flags 8042, mode 81a4 opened 0 00000080:00200000:2.0:1775764832.329627:0:66166:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f26.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000002:00010000:2.0:1775764832.329730:0:66166:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f26.sanity-pcc,[0x200003ab4:0x19:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0100103 00010000:00010000:2.0:1775764832.329744:0:66166:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052b22 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764832.329801:0:66166:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052b22 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764832.329809:0:66166:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764832.329811:0:66166:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740a00/0x73cfd86cfc17c166 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052b22 expref: -99 pid: 66140 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764832.329848:0:66166:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c189 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66166 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764832.329856:0:66166:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c189 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66166 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764832.329866:0:66166:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764832.329876:0:66166:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7480 x1862022098685952/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764832.329886:0:66166:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c189 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66166 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764832.329907:0:66166:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:66166:1862022098685952:192.168.202.128@tcp:101:multiop.0 00000100:00100000:2.0:1775764832.330336:0:66166:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764832.340763:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2378:1862022098685440:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764832.340779:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098685696:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764832.340791:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2378:1862022098685568:192.168.202.128@tcp:400:kworker.0 00010000:00010000:2.0:1775764833.209327:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764834.079703:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764834.231716:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764836.189208:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200080:1.0:1775764836.190078:0:66194:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086696 arg=7fb4bc000d20 00800000:00000080:1.0:1775764836.190106:0:66194:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=c0086696 len=4128 karg=ffff8b23ab528000 uarg=0000000000000000 00000002:00000080:1.0:1775764836.190117:0:66194:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=c0086696 len=4128 karg=ffff8b23ab528000 uarg=0000000000000000 00000002:00000080:1.0:1775764836.190124:0:66194:0:(mdc_request.c:1738:mdc_ioc_fid2path()) path get [0x200003ab4:0x17:0x0] from 18446744073709551615 #0 00000100:00100000:1.0:1775764836.190168:0:66194:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098686848:192.168.202.128@tcp:53:lhsmtool_posix.0 00000100:00100000:1.0:1775764836.190534:0:66194:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1c480 going to sleep for 11 seconds 00000100:00100000:1.0:1775764836.209981:0:66194:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098686848:192.168.202.128@tcp:53:lhsmtool_posix.0 00000002:00000080:1.0:1775764836.210012:0:66194:0:(mdc_request.c:1758:mdc_ioc_fid2path()) path got [0x200000007:0x1:0x0] from 18446744073709551615 #0: f26.sanity-pcc 00000080:00200080:1.0:1775764836.210155:0:66194:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c01066f8 arg=10f49e8 00800000:00000080:1.0:1775764836.210167:0:66194:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=c01066f8 len=16 karg=ffffa60b024efcf0 uarg=ffffa60b024efcc0 00000080:00200080:1.0:1775764836.210385:0:66194:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000002:0x2:0x0](ffff8b2390ec5348) cmd=c1406916 arg=7fb4bc000d20 00000100:00100000:1.0:1775764836.210447:0:66194:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098686976:192.168.202.128@tcp:34:lhsmtool_posix.0 00000100:00100000:1.0:1775764836.210841:0:66194:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1c480 going to sleep for 11 seconds 00000100:00100000:1.0:1775764836.223694:0:66194:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098686976:192.168.202.128@tcp:34:lhsmtool_posix.0 00000080:00200000:1.0:1775764836.223783:0:66194:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000002:0x2:0x0](ffff8b2390ec5348), inode mode 4040 mask 1 00000080:00200000:1.0:1775764836.223802:0:66194:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=0x200003ab4:0x17:0x0, dir=[0x200000002:0x2:0x0](ffff8b2390ec5348), file ffff8b238629b200, open_flags 68800, mode 0 opened 0 00000080:00200000:1.0:1775764836.223810:0:66194:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=0x200003ab4:0x17:0x0, dir=[0x200000002:0x2:0x0](ffff8b2390ec5348), intent=open 00000002:00010000:1.0:1775764836.223835:0:66194:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x17:0x0]) in obj [0x200000002:0x2:0x0], intent: open flags 041504001 00010000:00010000:1.0:1775764836.223893:0:66194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c19e lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66194 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.223902:0:66194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c19e lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66194 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764836.223923:0:66194:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764836.223931:0:66194:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fc000 x1862022098687232/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764836.223947:0:66194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c19e lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66194 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764836.223965:0:66194:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098687232:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:1.0:1775764836.224121:0:66194:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1c480 going to sleep for 11 seconds 00000100:00100000:1.0:1775764836.246006:0:66194:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098687232:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:1.0:1775764836.246031:0:66194:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c19e lrc: 4/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66194 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.246045:0:66194:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c19e lrc: 4/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66194 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.246052:0:66194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c19e lrc: 4/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66194 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.246059:0:66194:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c19e lrc: 3/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66194 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.246068:0:66194:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c19e lrc: 4/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66194 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.246074:0:66194:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8740800 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764836.246079:0:66194:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c19e lrc: 4/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 66194 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.246085:0:66194:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c19e lrc: 4/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 66194 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.246093:0:66194:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c19e lrc: 5/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 66194 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.246100:0:66194:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1775764836.246107:0:66194:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c19e lrc: 3/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66194 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.246113:0:66194:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740800/0x73cfd86cfc17c19e lrc: 0/0,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66194 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764836.246132:0:66194:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23864fc000 x1862022098687232/t34359739337(34359739337) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764852 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764836.246146:0:66194:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fc000 x1862022098687232/t34359739337(34359739337) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764852 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764836.246162:0:66194:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000002:0x2:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764836.246263:0:66194:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0x17:0x0], updating i_size 16 00000080:00200000:1.0:1775764836.246267:0:66194:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0x17:0x0](ffff8b23899c63c8): rc = 0 00000080:00010000:1.0:1775764836.246272:0:66194:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x17:0x0](ffff8b23899c63c8) for lock 0x0 00000080:00200008:1.0:1775764836.246295:0:66194:0:(file.c:1057:ll_file_open()) START file 0x200003ab4:0x17:0x0:[0x200003ab4:0x17:0x0](ffff8b23899c63c8), flags 1504000 00000002:00100000:1.0:1775764836.246315:0:66194:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fc000 x1862022098687232/t34359739337(34359739337) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764852 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764836.246391:0:66194:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x17:0x0](ffff8b23899c63c8), inode mode 81a4 mask 44 00000080:00200080:1.0:1775764836.246421:0:66194:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086696 arg=7fb4bc000d20 00800000:00000080:1.0:1775764836.246437:0:66194:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=c0086696 len=4128 karg=ffff8b23ab528000 uarg=0000000000000000 00000002:00000080:1.0:1775764836.246444:0:66194:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=c0086696 len=4128 karg=ffff8b23ab528000 uarg=0000000000000000 00000002:00000080:1.0:1775764836.246449:0:66194:0:(mdc_request.c:1738:mdc_ioc_fid2path()) path get [0x200003ab4:0x17:0x0] from 18446744073709551615 #0 00000100:00100000:1.0:1775764836.246575:0:66194:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098687360:192.168.202.128@tcp:53:lhsmtool_posix.0 00000100:00100000:1.0:1775764836.246664:0:66194:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1c480 going to sleep for 11 seconds 00000100:00100000:1.0:1775764836.254900:0:66194:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098687360:192.168.202.128@tcp:53:lhsmtool_posix.0 00000002:00000080:1.0:1775764836.254931:0:66194:0:(mdc_request.c:1758:mdc_ioc_fid2path()) path got [0x200000007:0x1:0x0] from 18446744073709551615 #0: f26.sanity-pcc 00000080:00200000:1.0:1775764836.255008:0:66194:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764836.255032:0:66194:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764836.255049:0:66194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17c190 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052bbc expref: -99 pid: 66193 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764836.255066:0:66194:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17c190 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052bbc expref: -99 pid: 66193 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764836.255075:0:66194:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764836.255083:0:66194:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764836.255087:0:66194:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c190 00000080:00010000:1.0:1775764836.255095:0:66194:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c190 from it ffffa60b024efbf8 00010000:00010000:1.0:1775764836.255099:0:66194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17c190 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052bbc expref: -99 pid: 66193 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764836.255108:0:66194:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17c190 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052bbc expref: -99 pid: 66193 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764836.255125:0:66194:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764836.255146:0:66194:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764836.255152:0:66194:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764836.255158:0:66194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17c190 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052bbc expref: -99 pid: 66193 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764836.255166:0:66194:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17c190 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052bbc expref: -99 pid: 66193 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764836.255173:0:66194:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764836.255176:0:66194:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764836.255178:0:66194:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c190 00000080:00010000:1.0:1775764836.255332:0:66194:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c190 from it ffffa60b024efb38 00010000:00010000:1.0:1775764836.255337:0:66194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17c190 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052bbc expref: -99 pid: 66193 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764836.255765:0:66194:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17c190 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052bbc expref: -99 pid: 66193 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764836.255776:0:66194:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:1.0:1775764836.255783:0:66194:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=. :VOLATILE:0000:2ED5EAD9:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b238629b380, open_flags 280c2, mode 8180 opened 0 00000100:00100000:2.0:1775764836.264887:0:66194:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098687616:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764836.264917:0:66194:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1a5 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c02 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.264948:0:66194:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1a5 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c02 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.264956:0:66194:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1a5 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c02 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.264969:0:66194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1a5 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c02 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.264981:0:66194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1a5 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c02 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.264995:0:66194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1a5 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c02 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.265004:0:66194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1a5 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c02 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.265011:0:66194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1a5 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c02 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764836.265023:0:66194:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8b2382daed80 x1862022098687616/t34359739338(34359739338) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1088/664 e 0 to 0 dl 1775764852 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764836.265035:0:66194:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382daed80 x1862022098687616/t34359739338(34359739338) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1088/664 e 0 to 0 dl 1775764852 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764836.265048:0:66194:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1a5 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c02 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764836.265061:0:66194:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1a5 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c02 expref: -99 pid: 66194 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.265072:0:66194:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/26 (0 0) 00000080:00200000:2.0:1775764836.265079:0:66194:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764836.265091:0:66194:0:(namei.c:1620:ll_create_it()) VFS Op:name=. :VOLATILE:0000:2ED5EAD9:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000080:00200000:2.0:1775764836.265119:0:66194:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab4:0x1a:0x0](ffff8b2390ec63c8) 00000080:00200000:2.0:1775764836.265820:0:66194:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1a:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1775764836.265835:0:66194:0:(namei.c:1586:ll_create_node()) setting l_ast_data to inode [0x200003ab4:0x1a:0x0](ffff8b2390ec63c8) 00000080:00010000:2.0:1775764836.265838:0:66194:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1a:0x0](ffff8b2390ec63c8) for lock 0x73cfd86cfc17c1a5 00000080:00010000:2.0:1775764836.265847:0:66194:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1a:0x0](ffff8b2390ec63c8) 00000080:00200008:2.0:1775764836.265862:0:66194:0:(file.c:1057:ll_file_open()) START file . :VOLATILE:0000:2ED5EAD9:fd=09:[0x200003ab4:0x1a:0x0](ffff8b2390ec63c8), flags 500302 00000002:00100000:2.0:1775764836.265883:0:66194:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382daed80 x1862022098687616/t34359739338(34359739338) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 608/664 e 0 to 0 dl 1775764852 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764836.265920:0:66194:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c1a5 from it ffff8b23891e8c60 00010000:00010000:2.0:1775764836.265924:0:66194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1a5 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052c02 expref: -99 pid: 66194 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.265934:0:66194:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1a5 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052c02 expref: -99 pid: 66194 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764836.265973:0:66194:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x1a:0x0](ffff8b2390ec63c8), inode mode 8180 mask 40 00000080:00200000:2.0:1775764836.266006:0:66194:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764836.266020:0:66194:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764836.266062:0:66194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66194 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.266070:0:66194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66194 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.266078:0:66194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66194 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764836.266098:0:66194:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098687744:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764836.266197:0:66194:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942300 going to sleep for 11 seconds 00000100:00100000:2.0:1775764836.268942:0:66194:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098687744:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764836.268970:0:66194:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c17 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.268982:0:66194:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c17 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.268996:0:66194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c17 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.269007:0:66194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c17 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.269021:0:66194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c17 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.269030:0:66194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c17 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.269037:0:66194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c17 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764836.269048:0:66194:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea4380 x1862022098687744/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764852 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764836.269061:0:66194:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c17 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.269073:0:66194:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764836.269079:0:66194:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764836.269108:0:66194:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764836.269114:0:66194:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c1ac 00000080:00010000:2.0:1775764836.269119:0:66194:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764836.269121:0:66194:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c1ac from it ffffa60b024efc10 00010000:00010000:2.0:1775764836.269125:0:66194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c17 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.269133:0:66194:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c17 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764836.269179:0:66194:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764836.269204:0:66194:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=. :VOLATILE:0000:2ED5EAD9:fd=09, flags=0 00000080:00200000:2.0:1775764836.269211:0:66194:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764836.269225:0:66194:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764836.269241:0:66194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c17 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764836.269252:0:66194:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c17 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764836.269258:0:66194:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764836.269263:0:66194:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764836.269266:0:66194:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c1ac 00000080:00010000:2.0:1775764836.269269:0:66194:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c1ac from it ffffa60b024efd70 00010000:00010000:2.0:1775764836.269272:0:66194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c17 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764836.269279:0:66194:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c17 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764836.269288:0:66194:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:2.0:1775764836.269293:0:66194:0:(namei.c:2405:ll_unlink()) VFS Op:name=. :VOLATILE:0000:2ED5EAD9:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08) 00010000:00010000:2.0:1775764836.269370:0:66194:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1ac lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052c17 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764836.269397:0:66194:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1a5 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052c02 expref: -99 pid: 66194 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764836.269418:0:66194:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1a:0x0]: losing layout lock 00000100:00100000:2.0:1775764836.314191:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098688640:192.168.202.128@tcp:2:lhsmtool_posix.0 00000100:00100000:2.0:1775764836.335261:0:2379:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b2389398000: committing for last_committed 34359738459 gen 1 00000100:00100000:2.0:1775764836.335292:0:2379:0:(client.c:3126:ptlrpc_free_committed()) @@@ stopping search req@ffff8b23864f1500 x1862022098688640/t34359738462(34359738462) o2->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 440/432 e 0 to 0 dl 1775764852 ref 2 fl Rpc:RQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764836.335326:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098688640:192.168.202.128@tcp:2:lhsmtool_posix.0 00000100:00100000:1.0:1775764836.367419:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098689408:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:1.0:1775764836.377744:0:2377:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000000778f3d6/0x73cfd86cfc17c1c8 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.377771:0:2377:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000000778f3d6/0x73cfd86cfc17c1c8 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c56 expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.377786:0:2377:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000000778f3d6/0x73cfd86cfc17c1c8 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c56 expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.377810:0:2377:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000000778f3d6/0x73cfd86cfc17c1c8 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c56 expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1775764836.377832:0:2377:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000000778f3d6/0x73cfd86cfc17c1c8 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052c56 expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1775764836.377843:0:2377:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1775764836.377903:0:2377:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000000778f3d6/0x73cfd86cfc17c1c8 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052c56 expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.377912:0:2377:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000000778f3d6/0x73cfd86cfc17c1c8 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052c56 expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.377919:0:2377:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000000778f3d6/0x73cfd86cfc17c1c8 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052c56 expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.377927:0:2377:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000000778f3d6/0x73cfd86cfc17c1c8 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052c56 expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1775764836.377947:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098689408:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764836.398268:0:66194:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098689664:192.168.202.128@tcp:57:lhsmtool_posix.0 00000080:00200008:2.0:1775764836.398526:0:66194:0:(file.c:2612:do_file_write_iter()) START file . :VOLATILE:0000:2ED5EAD9:fd=09:[0x200003ab4:0x1a:0x0], ppos: 0, count: 16 00000080:00200000:2.0:1775764836.398545:0:66194:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 16 bytes 00000080:00200000:2.0:1775764836.398552:0:66194:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 16 00000080:00200000:2.0:1775764836.398603:0:66194:0:(file.c:2058:ll_file_io_generic()) . :VOLATILE:0000:2ED5EAD9:fd=09: write ppos: 0, bytes: 16 00000080:00200000:2.0:1775764836.398610:0:66194:0:(file.c:1815:ll_io_set_mirror()) . :VOLATILE:0000:2ED5EAD9:fd=09: desiginated mirror: 0 00000020:00200000:2.0:1775764836.398619:0:66194:0:(cl_io.c:214:cl_io_rw_init()) header@0000000058e00b41[0x110, 1, [0x200003ab4:0x1a:0x0] hash] 00000020:00200000:2.0:1775764836.398622:0:66194:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 16) 0 0 00000080:00200000:2.0:1775764836.398627:0:66194:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764836.398643:0:66194:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:2.0:1775764836.398650:0:66194:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 16) 16, 16 00020000:00200000:2.0:1775764836.398654:0:66194:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764836.398685:0:66194:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1775764836.398705:0:66194:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764836.398710:0:66194:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c84b0, osc lock ffff8b238a3f8be0, flags 0x0 00010000:00010000:2.0:1775764836.398729:0:66194:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2386938000 type 11 mode 2 res 10737419264/285 (0 4095) 00010000:00010000:2.0:1775764836.398779:0:66194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.398791:0:66194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.398802:0:66194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764836.398823:0:66194:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098689792:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764836.398974:0:66194:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942300 going to sleep for 11 seconds 00000100:00100000:1.0:1775764836.404751:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119076480 00000100:00100000:1.0:1775764836.404765:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119076480 00000100:00100000:1.0:1775764836.404853:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 286 00000100:00100000:1.0:1775764836.404863:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daf480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42946:x1862022119076480:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764836.404882:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000000778f3d6/0x73cfd86cfc17c1c8 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e03052c56 expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764836.404977:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764836.404984:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daf480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42946:x1862022119076480:12345-192.168.202.128@tcp:104: Request processed in 123us (565us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764836.404991:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 286 00000100:00100000:2.0:1775764836.413230:0:66194:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098689792:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764836.413259:0:66194:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.413274:0:66194:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.413282:0:66194:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764836.414046:0:46140:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764836.414067:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119076608 00000100:00100000:1.0:1775764836.414075:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119076608 00000100:00100000:1.0:1775764836.414137:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 287 00000100:00100000:1.0:1775764836.414146:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23b9ff1050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42309:x1862022119076608:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764836.414157:0:46140:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.414168:0:46140:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.414177:0:46140:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.414186:0:46140:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.414195:0:46140:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.414205:0:46140:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.414307:0:46140:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b2380031200) 00000100:00100000:1.0:1775764836.414365:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23b9ff1050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42309:x1862022119076608:12345-192.168.202.128@tcp:105: Request processed in 219us (1295us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764836.414384:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 287 00010000:00010000:2.0:1775764836.414555:0:66194:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.414573:0:66194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.414595:0:66194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1775764836.414606:0:66194:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000020000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.414618:0:66194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.414625:0:66194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764836.414651:0:66194:0:(vvp_io.c:1286:vvp_io_write_start()) . :VOLATILE:0000:2ED5EAD9:fd=09: write [0, 16) append 0 append lockpos 0 00000080:00200000:2.0:1775764836.414668:0:66194:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 16 bytes 00000080:00200000:2.0:1775764836.414725:0:66194:0:(rw26.c:858:ll_write_end()) pos 0, len 16, copied 16 00000080:00200000:2.0:1775764836.414734:0:66194:0:(rw26.c:916:ll_write_end()) page@00000000789db01c[3 000000007a6af3d9 2 1 00000000b944163c] 00000080:00200000:2.0:1775764836.414738:0:66194:0:(rw26.c:916:ll_write_end()) vmpage @00000000c508d6a3 17ffffc0001001 3:0 ffff8b23baeae000 0 lru 00000080:00200000:2.0:1775764836.414745:0:66194:0:(rw26.c:916:ll_write_end()) osc-page@00000000561d62ec 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 000000008819327c 000000003d8bbb9f > 3< 0 0 > 4< 0 0 8 15335424 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764836.414747:0:66194:0:(rw26.c:916:ll_write_end()) end page@00000000789db01c 00000080:00200000:2.0:1775764836.414748:0:66194:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764836.414753:0:66194:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 16 prio 0 00000008:00010000:2.0:1775764836.414775:0:66194:0:(osc_cache.c:848:osc_extent_find()) ### extent: 00000000451ad45b ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764836.414794:0:66194:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2390ec6578, count 1, dirtied 1 00000080:00200000:2.0:1775764836.414798:0:66194:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 16 bytes, tot: 16 00000080:00200000:2.0:1775764836.414801:0:66194:0:(vvp_io.c:1353:vvp_io_write_start()) . :VOLATILE:0000:2ED5EAD9:fd=09: write bytes 0, result: 16 00000020:00010000:2.0:1775764836.414816:0:66194:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b5d8 (ffff8b23812ec998/0) at cl_lock_release():245 00000020:00010000:2.0:1775764836.414819:0:66194:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b5d8 (ffff8b23812ec998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764836.414823:0:66194:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c84b0 (ffff8b23812ec7e8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764836.414827:0:66194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.414837:0:66194:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1775764836.414852:0:66194:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b5d8 (ffff8b23812ec998/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764836.414855:0:66194:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c84b0 (ffff8b23812ec998/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764836.414862:0:66194:0:(vvp_io.c:517:vvp_io_advance()) advancing 16 bytes 00000080:00200000:2.0:1775764836.414869:0:66194:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:2.0:1775764836.414895:0:66194:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764836.414898:0:66194:0:(file.c:2206:ll_file_io_generic()) . :VOLATILE:0000:2ED5EAD9:fd=09: 2 io complete with rc: 0, result: 16, restart: 0 00000080:00200000:2.0:1775764836.414905:0:66194:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1a:0x0] updating i_size 16 i_blocks 1 00000080:00200000:2.0:1775764836.414924:0:66194:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 16 00000080:00200000:2.0:1775764836.415115:0:66194:0:(file.c:5304:ll_fsync()) VFS Op:inode=[0x200003ab4:0x1a:0x0](ffff8b2390ec63c8), start 0, end 9223372036854775807, datasync 0 00000080:00200000:2.0:1775764836.415145:0:66194:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764836.415154:0:66194:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:2.0:1775764836.415179:0:66194:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 00000000451ad45b ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1775764836.415193:0:66194:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000451ad45b ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764836.415248:0:66194:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00100000:2.0:1775764836.415315:0:66194:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea6300 - object 0x280000400:285 offset 0<>16 00000100:00100000:2.0:1775764836.415368:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764836.415384:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098690048:192.168.202.128@tcp:4:lhsmtool_posix.0 00000008:00010000:2.0:1775764836.420168:0:2378:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 00000000451ad45b ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:2.0:1775764836.424351:0:2378:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000451ad45b ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000001a2d6766/0x73cfd86cfc17c1cf lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03052c5d expref: -99 pid: 66194 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1775764836.424406:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098690048:192.168.202.128@tcp:4:lhsmtool_posix.0 00000100:00100000:1.0:1775764836.525993:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098690304:192.168.202.128@tcp:16:lhsmtool_posix.0 00000080:00200000:2.0:1775764836.565798:0:66194:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1a:0x0] updating i_size 16 i_blocks 11 00000080:00200000:2.0:1775764836.565834:0:66194:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:2.0:1775764836.566770:0:66194:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x1a:0x0](ffff8b2390ec63c8) cmd=c03066b0 arg=7fb4c3947ac0 00000080:00200008:2.0:1775764836.566786:0:66194:0:(llite_lib.c:2274:ll_setattr_raw()) START file . :VOLATILE:0000:2ED5EAD9:fd=09:[0x200003ab4:0x1a:0x0](ffff8b2390ec63c8) current size 16, valid attrs 1f0, mode 0, uid 0, gid 0, new size 0, atime 1775764831.000000000, mtime 1775764831.000000000, ctime 1775764831.000000000 00010000:00010000:2.0:1775764836.566827:0:66194:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17c1c1 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052c4f expref: -99 pid: 66194 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764836.566855:0:66194:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17c1ba lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052c41 expref: -99 pid: 66194 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764836.566922:0:66194:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17c1c1 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052c4f expref: -99 pid: 66194 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764836.566929:0:66194:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764836.566933:0:66194:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17c1c1 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052c4f expref: -99 pid: 66194 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000020:00100000:2.0:1775764836.566954:0:66194:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764836.566964:0:66194:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4000 x1862022098690560/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764836.566990:0:66194:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098690560:192.168.202.128@tcp:36:lhsmtool_posix.0 00000100:00100000:2.0:1775764836.567100:0:66194:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942300 going to sleep for 11 seconds 00000100:00100000:2.0:1775764836.571771:0:66194:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 34359739340 gen 1 00000100:00100000:2.0:1775764836.571790:0:66194:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b2382daed80 x1862022098687616/t34359739338(34359739338) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 608/664 e 0 to 0 dl 1775764852 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764836.571813:0:66194:0:(client.c:3126:ptlrpc_free_committed()) @@@ stopping search req@ffff8b23b9ea4000 x1862022098690560/t34359739341(34359739341) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 488/456 e 0 to 0 dl 1775764852 ref 3 fl Rpc:RQU/204/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764836.571824:0:66194:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098690560:192.168.202.128@tcp:36:lhsmtool_posix.0 00000080:00200000:2.0:1775764836.572042:0:66194:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764836.572056:0:66194:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764836.581020:0:66194:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00800000:00000080:2.0:1775764836.581047:0:66194:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d4 len=384 karg=ffff8b23baec5600 uarg=0000000000000000 00000002:00000080:2.0:1775764836.581058:0:66194:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d4 len=384 karg=ffff8b23baec5600 uarg=0000000000000000 00000020:00100000:2.0:1775764836.581086:0:66194:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 55, max 7 00000100:00100000:2.0:1775764836.581094:0:66194:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5880 x1862022098690944/t0(0) o55->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 472/224 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764836.581119:0:66194:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098690944:192.168.202.128@tcp:55:lhsmtool_posix.0 00000100:00100000:2.0:1775764836.581632:0:66194:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942300 going to sleep for 11 seconds 00000100:00100000:1.0:1775764836.586446:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764836.587979:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764836.587996:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c1b3 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03052c33 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764836.588009:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5f200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764836.588015:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c1b3 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03052c33 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764836.588024:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c1b3 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03052c33 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764836.588033:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c1b3 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x57b8ff0e03052c33 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764836.588092:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c1b3 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03052c33 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764836.588099:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764836.588130:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c1b3 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03052c33 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764836.588138:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c1b3 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03052c33 expref: -99 pid: 66194 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764836.588151:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764836.588153:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764836.588154:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764836.592162:0:66194:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66194:1862022098690944:192.168.202.128@tcp:55:lhsmtool_posix.0 00000080:00200008:2.0:1775764836.592200:0:66194:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file . :VOLATILE:0000:2ED5EAD9:fd=09:[0x200003ab4:0x1a:0x0](ffff8b2390ec63c8) current size 16, valid attrs 1f0, mode 0, uid 0, gid 0, new size 0, atime 1775764831.000000000, mtime 1775764831.000000000, ctime 1775764831.000000000, rc 0 00000080:00200080:2.0:1775764836.592233:0:66194:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=400866d7 arg=7fb4bc000b78 00000080:00200000:2.0:1775764836.592427:0:66194:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764836.592439:0:66194:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:2.0:1775764836.592587:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098691200:192.168.202.128@tcp:1:lhsmtool_posix.0 00000100:00100000:1.0:1775764836.596447:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:2.0:1775764836.597812:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098691328:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764836.603579:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098691328:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764836.605059:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119077120 00000100:00100000:1.0:1775764836.605068:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119077120 00000100:00100000:1.0:1775764836.605119:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 290 00000100:00100000:1.0:1775764836.605129:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dad180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42326:x1862022119077120:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764836.605145:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17c1ba lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03052c41 expref: -99 pid: 66194 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764836.605219:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764836.605227:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dad180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42326:x1862022119077120:12345-192.168.202.128@tcp:104: Request processed in 100us (263us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764836.605233:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 290 00010000:00010000:2.0:1775764836.605583:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764836.605591:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17c1ba lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03052c41 expref: -99 pid: 66194 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764836.605600:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5fa00 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764836.605605:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17c1ba lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052c41 expref: -99 pid: 66194 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764836.605613:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17c1ba lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052c41 expref: -99 pid: 66194 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764836.605622:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17c1ba lrc: 3/0,0 mode: CR/CR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03052c41 expref: -99 pid: 66194 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764836.605670:0:46150:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1a:0x0]: losing layout lock 00010000:00010000:2.0:1775764836.605721:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17c1ba lrc: 2/0,0 mode: --/CR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052c41 expref: -99 pid: 66194 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764836.605727:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764836.605753:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17c1ba lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052c41 expref: -99 pid: 66194 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764836.605761:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17c1ba lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x1a:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052c41 expref: -99 pid: 66194 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764836.605783:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764836.605785:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764836.605787:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764836.614899:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119077504 00000100:00100000:1.0:1775764836.614910:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119077504 00000100:00100000:1.0:1775764836.614962:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 291 00000100:00100000:1.0:1775764836.614971:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dac000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42326:x1862022119077504:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764836.614987:0:46140:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17c197 lrc: 5/1,0 mode: --/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052bca expref: -99 pid: 66193 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.615005:0:46140:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17c197 lrc: 5/1,0 mode: --/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052bca expref: -99 pid: 66193 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.615014:0:46140:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17c197 lrc: 5/1,0 mode: --/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052bca expref: -99 pid: 66193 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.615022:0:46140:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17c197 lrc: 5/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e03052bca expref: -99 pid: 66193 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.615030:0:46140:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17c197 lrc: 6/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e03052bca expref: -99 pid: 66193 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.615038:0:46140:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17c197 lrc: 6/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x80000000000 nid: local remote: 0x57b8ff0e03052bca expref: -99 pid: 66193 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764836.615064:0:46140:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b23b9c5fe00) 00000100:00100000:1.0:1775764836.615120:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dac000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42326:x1862022119077504:12345-192.168.202.128@tcp:105: Request processed in 150us (330us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764836.615127:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 291 00000080:00200000:2.0:1775764836.649105:0:66196:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764836.649133:0:66196:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764836.649194:0:66196:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66196 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.649205:0:66196:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66196 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764836.649215:0:66196:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66196 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764836.649241:0:66196:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66196:1862022098692480:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764836.649351:0:66196:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942300 going to sleep for 11 seconds 00000100:00100000:1.0:1775764837.659626:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119077632 00000100:00100000:1.0:1775764837.659637:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119077632 00000100:00100000:1.0:1775764837.659699:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 292 00000100:00100000:1.0:1775764837.659709:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daed80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42356:x1862022119077632:12345-192.168.202.128@tcp:107:hsm_cdtr.0 00000100:00100000:1.0:1775764837.659814:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daed80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42356:x1862022119077632:12345-192.168.202.128@tcp:107:hsm_cdtr.0 Request processed in 105us (309us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764837.659825:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 292 00010000:00010000:2.0:1775764837.692391:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764837.692412:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1e4 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03052cc6 expref: -99 pid: 66208 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.692428:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2381497200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764837.692521:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1e4 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03052cc6 expref: -99 pid: 66208 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.692540:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1e4 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03052cc6 expref: -99 pid: 66208 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.692579:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1e4 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052cc6 expref: -99 pid: 66208 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.692610:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c1dd lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052cb8 expref: -99 pid: 66196 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764837.692660:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1e4 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052cc6 expref: -99 pid: 66208 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.692666:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764837.692712:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1e4 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052cc6 expref: -99 pid: 66208 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.692720:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1e4 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052cc6 expref: -99 pid: 66208 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764837.692739:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764837.692741:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764837.692743:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764837.692755:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764837.692775:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098693760:192.168.202.128@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764839.351630:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1775764840.012110:0:66222:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x17:0x0] updating i_size 16 i_blocks 11 00000020:00010000:2.0:1775764840.012118:0:66222:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2389eaa910 (ffff8b238a3ff758/0) at cl_lock_release():245 00000020:00010000:2.0:1775764840.012122:0:66222:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2389eaa910 (ffff8b238a3ff758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764840.012127:0:66222:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2391066fb0 (ffff8b238a3ffc68/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764840.012133:0:66222:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2389eaa910 (ffff8b238a3ff758/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764840.012135:0:66222:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2391066fb0 (ffff8b238a3ff758/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764840.012171:0:66222:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x17:0x0] ignore/verify layout 0/1, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764840.012224:0:66222:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764840.012242:0:66222:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764840.012256:0:66222:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764840.012279:0:66222:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764840.012288:0:66222:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764840.012295:0:66222:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764840.012298:0:66222:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c1d6 00000080:00010000:2.0:1775764840.012304:0:66222:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c1d6 from it ffffa60b024afbf8 00010000:00010000:2.0:1775764840.012307:0:66222:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764840.012314:0:66222:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764840.012323:0:66222:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764840.012330:0:66222:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f26.sanity-pcc, flags=265 00000080:00200000:2.0:1775764840.012336:0:66222:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x17:0x0](ffff8b23899c63c8), inode mode 81a4 mask 44 00000080:00200008:2.0:1775764840.012344:0:66222:0:(file.c:1057:ll_file_open()) START file f26.sanity-pcc:[0x200003ab4:0x17:0x0](ffff8b23899c63c8), flags 100000 00000002:00010000:2.0:1775764840.012359:0:66222:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x17:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 04240100001 00010000:00010000:2.0:1775764840.012404:0:66222:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1f2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66222 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764840.012413:0:66222:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1f2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66222 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764840.012436:0:66222:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764840.012445:0:66222:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5880 x1862022098694400/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764840.012494:0:66222:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1f2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66222 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764840.012517:0:66222:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66222:1862022098694400:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764840.012715:0:66222:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942300 going to sleep for 11 seconds 00000100:00100000:2.0:1775764840.020214:0:66222:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66222:1862022098694400:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764840.020254:0:66222:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1f2 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ce9 expref: -99 pid: 66222 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764840.020276:0:66222:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1f2 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ce9 expref: -99 pid: 66222 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764840.020288:0:66222:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1f2 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052ce9 expref: -99 pid: 66222 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764840.020300:0:66222:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1f2 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ce9 expref: -99 pid: 66222 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764840.020316:0:66222:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23b9ea5880 x1862022098694400/t34359739353(34359739353) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764856 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764840.020331:0:66222:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea5880 x1862022098694400/t34359739353(34359739353) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764856 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764840.020344:0:66222:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1f2 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ce9 expref: -99 pid: 66222 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764840.020357:0:66222:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c1f2 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ce9 expref: -99 pid: 66222 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764840.020368:0:66222:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/23 (0 0) 00000080:00200000:2.0:1775764840.020375:0:66222:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764840.162341:0:66232:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764840.162368:0:66232:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764840.162384:0:66232:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764840.162398:0:66232:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764840.162403:0:66232:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764840.162409:0:66232:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764840.162412:0:66232:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c1d6 00000080:00010000:2.0:1775764840.162418:0:66232:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c1d6 from it ffffa60b02547bb0 00010000:00010000:2.0:1775764840.162422:0:66232:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764840.162431:0:66232:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764840.162446:0:66232:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764840.162479:0:66232:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764840.162503:0:66232:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffdc2b19f10 00000100:00100000:2.0:1775764840.162575:0:66232:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66232:1862022098695168:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764840.162676:0:66232:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942300 going to sleep for 11 seconds 00000100:00100000:2.0:1775764840.167376:0:66232:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66232:1862022098695168:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764840.167443:0:66232:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:1.0:1775764840.257135:0:66239:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764840.257165:0:66239:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764840.257185:0:66239:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764840.257205:0:66239:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764840.257214:0:66239:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764840.257223:0:66239:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764840.257227:0:66239:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c1d6 00000080:00010000:1.0:1775764840.257235:0:66239:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c1d6 from it ffffa60b024dfbf8 00010000:00010000:1.0:1775764840.257239:0:66239:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764840.257247:0:66239:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764840.257638:0:66239:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764840.257661:0:66239:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f26.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b23864d0900, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1775764840.257668:0:66239:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f26.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open 00000002:00010000:1.0:1775764840.257684:0:66239:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f26.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100001 00010000:00010000:1.0:1775764840.257739:0:66239:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66239 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764840.257747:0:66239:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66239 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764840.257758:0:66239:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764840.257770:0:66239:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fdf80 x1862022098695424/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764840.257780:0:66239:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66239 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764840.257803:0:66239:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:66239:1862022098695424:192.168.202.128@tcp:101:cat.0 00000100:00100000:1.0:1775764840.259214:0:66239:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2387d1c6c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764840.280490:0:66239:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:66239:1862022098695424:192.168.202.128@tcp:101:cat.0 00010000:00010000:2.0:1775764840.280539:0:66239:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d36 expref: -99 pid: 66239 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764840.280558:0:66239:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d36 expref: -99 pid: 66239 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764840.280566:0:66239:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d36 expref: -99 pid: 66239 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764840.280579:0:66239:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052d36 expref: -99 pid: 66239 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764840.280591:0:66239:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d36 expref: -99 pid: 66239 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764840.280604:0:66239:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052d36 expref: -99 pid: 66239 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764840.280618:0:66239:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052d36 expref: -99 pid: 66239 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764840.280626:0:66239:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052d36 expref: -99 pid: 66239 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764840.280637:0:66239:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23864fdf80 x1862022098695424/t34359739356(34359739356) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/864 e 0 to 0 dl 1775764856 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764840.280649:0:66239:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fdf80 x1862022098695424/t34359739356(34359739356) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/864 e 0 to 0 dl 1775764856 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764840.280663:0:66239:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d36 expref: -99 pid: 66239 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764840.280679:0:66239:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d36 expref: -99 pid: 66239 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764840.280689:0:66239:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/23 (0 0) 00000080:00200000:2.0:1775764840.280696:0:66239:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764840.280805:0:66239:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0x17:0x0](ffff8b23899c63c8): rc = 0 00000080:00200000:2.0:1775764840.280826:0:66239:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x17:0x0]: layout version change: 3 -> 3 00000080:00010000:2.0:1775764840.280836:0:66239:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x17:0x0](ffff8b23899c63c8) for lock 0x73cfd86cfc17c200 00000080:00010000:2.0:1775764840.280849:0:66239:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x17:0x0](ffff8b23899c63c8) 00000080:00200008:2.0:1775764840.280863:0:66239:0:(file.c:1057:ll_file_open()) START file f26.sanity-pcc:[0x200003ab4:0x17:0x0](ffff8b23899c63c8), flags 100000 00000002:00100000:2.0:1775764840.282515:0:66239:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fdf80 x1862022098695424/t34359739356(34359739356) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764856 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764840.282585:0:66239:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c200 from it ffff8b2389ed6240 00010000:00010000:2.0:1775764840.282593:0:66239:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052d36 expref: -99 pid: 66239 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764840.282607:0:66239:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052d36 expref: -99 pid: 66239 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764840.282646:0:66239:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x17:0x0](ffff8b23899c63c8), inode mode 81a4 mask 44 00000080:00200008:2.0:1775764840.282681:0:66239:0:(file.c:6226:ll_getattr_dentry()) START file f26.sanity-pcc:[0x200003ab4:0x17:0x0](ffff8b23899c63c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764840.282687:0:66239:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x17:0x0](ffff8b23899c63c8),name=f26.sanity-pcc 00000002:00010000:2.0:1775764840.282708:0:66239:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x17:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764840.282721:0:66239:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052d36 expref: -99 pid: 66239 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764840.282732:0:66239:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052d36 expref: -99 pid: 66239 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764840.282741:0:66239:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764840.282748:0:66239:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x17:0x0](ffff8b23899c63c8) 00000080:00010000:2.0:1775764840.282751:0:66239:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x17:0x0](ffff8b23899c63c8) for lock 0x73cfd86cfc17c200 00000080:00010000:2.0:1775764840.282755:0:66239:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c200 from it ffffa60b024dfcd8 00010000:00010000:2.0:1775764840.282758:0:66239:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052d36 expref: -99 pid: 66239 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764840.282764:0:66239:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052d36 expref: -99 pid: 66239 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200008:2.0:1775764840.282874:0:66239:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x17:0x0], ppos: 0, count: 4194304 00000080:00200008:2.0:1775764840.283144:0:66239:0:(file.c:2454:do_file_read_iter()) START file f26.sanity-pcc:[0x200003ab4:0x17:0x0], ppos: 16, count: 4194304 00000080:00200008:2.0:1775764840.283217:0:66239:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x17:0x0](ffff8b23899c63c8), flags 100000 00000002:00100000:2.0:1775764840.283268:0:66239:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fdf80 x1862022098695424/t34359739356(34359739356) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764856 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cat.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764840.283279:0:66239:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764840.283284:0:66239:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7b80 x1862022098695680/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764840.283306:0:66239:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:66239:1862022098695680:192.168.202.128@tcp:35:cat.0 00000100:00100000:2.0:1775764840.283400:0:66239:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b2386942300 going to sleep for 11 seconds 00000080:00200008:2.0:1775764840.739040:0:66166:0:(file.c:447:ll_file_release()) START file f26.sanity-pcc:[0x200003ab4:0x17:0x0](ffff8b23899c63c8), flags 100002 00000002:00100000:2.0:1775764840.739242:0:66166:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea7480 x1862022098685952/t34359739336(34359739336) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764848 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764840.739255:0:66166:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764840.739260:0:66166:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6300 x1862022098695936/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764840.739285:0:66166:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:66166:1862022098695936:192.168.202.128@tcp:35:multiop.0 00000100:00100000:2.0:1775764840.739396:0:66166:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654cc0 going to sleep for 11 seconds 00000080:00200000:2.0:1775764840.848788:0:66279:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764840.848820:0:66279:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764840.848840:0:66279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1775764840.848869:0:66279:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:2.0:1775764840.848879:0:66279:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764840.848887:0:66279:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764840.848890:0:66279:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c1d6 00000080:00010000:2.0:1775764840.848898:0:66279:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c1d6 from it ffffa60b02547bb0 00010000:00010000:2.0:1775764840.848902:0:66279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1775764840.848911:0:66279:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1775764840.848937:0:66279:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764840.848945:0:66279:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764840.849258:0:66279:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffc69f57840 00000080:00200008:2.0:1775764840.849290:0:66279:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00010000:00010000:2.0:1775764841.400570:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764842.555085:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2378:1862022098696064:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764842.555189:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098696320:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764842.555216:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098696576:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764842.555242:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1775764842.555247:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2378:1862022098696192:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764842.555267:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098696448:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764842.584630:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2378:1862022098696192:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764842.584651:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098696448:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764842.584660:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2378:1862022098696064:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764842.584669:0:2378:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 34359739358 gen 1 00000100:00100000:2.0:1775764842.584697:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098696320:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764842.584708:0:2378:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b2389398000: committing for last_committed 34359738465 gen 1 00000100:00100000:2.0:1775764842.584717:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098696576:192.168.202.128@tcp:400:kworker.0 00010000:00010000:2.0:1775764843.449758:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764846.520501:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764846.645128:0:65650:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:65650:1862022098697088:192.168.202.128@tcp:35:lhsmtool_posix.0 00000002:00100000:2.0:1775764846.645170:0:65650:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea6680 x1862022098661248/t34359739268(34359739268) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/4688 e 0 to 0 dl 1775764827 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1775764846.645250:0:65650:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 100000 00000080:00200008:1.0:1775764847.040251:0:66719:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764847.040265:0:66719:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:1.0:1775764847.040287:0:66719:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764847.040305:0:66719:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c207 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d4b expref: -99 pid: 66702 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764847.040323:0:66719:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c207 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d4b expref: -99 pid: 66702 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764847.040342:0:66719:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764847.040357:0:66719:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:1.0:1775764847.040361:0:66719:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17c207 00000080:00010000:1.0:1775764847.040367:0:66719:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c207 from it ffffa60b01d93cb8 00010000:00010000:1.0:1775764847.040371:0:66719:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c207 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052d4b expref: -99 pid: 66702 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764847.040379:0:66719:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c207 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d4b expref: -99 pid: 66702 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200008:1.0:1775764847.041015:0:66719:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764847.041022:0:66719:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764847.041034:0:66719:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764847.041045:0:66719:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764847.041056:0:66719:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:1.0:1775764847.041062:0:66719:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764847.041067:0:66719:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764847.041070:0:66719:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c1d6 00000080:00010000:1.0:1775764847.041075:0:66719:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c1d6 from it ffffa60b01d93cb8 00010000:00010000:1.0:1775764847.041078:0:66719:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1775764847.041087:0:66719:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000100:00100000:1.0:1775764847.681328:0:2376:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 34359739359 gen 1 00000100:00100000:1.0:1775764847.681684:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098697472:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764847.681707:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098697728:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764847.681736:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2376:1862022098697216:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764847.688127:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2376:1862022098697344:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764847.688144:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098697600:192.168.202.128@tcp:400:kworker.0 00010000:00010000:2.0:1775764848.567609:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764850.616450:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1775764853.165661:0:66980:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764853.165676:0:66980:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764853.165698:0:66980:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764853.165716:0:66980:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764853.165732:0:66980:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1775764853.165740:0:66980:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764853.165749:0:66980:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764853.165753:0:66980:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c1d6 00000080:00010000:1.0:1775764853.165760:0:66980:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c1d6 from it ffffa60b01e6bcb8 00010000:00010000:1.0:1775764853.165764:0:66980:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1775764853.165772:0:66980:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200008:1.0:1775764853.166539:0:66980:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1775764853.166546:0:66980:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:1.0:1775764853.166560:0:66980:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764853.166571:0:66980:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c207 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d4b expref: -99 pid: 66702 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764853.166584:0:66980:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c207 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d4b expref: -99 pid: 66702 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764853.166592:0:66980:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764853.166597:0:66980:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:1.0:1775764853.166600:0:66980:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17c207 00000080:00010000:1.0:1775764853.166605:0:66980:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c207 from it ffffa60b01e6bcb8 00010000:00010000:1.0:1775764853.166609:0:66980:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c207 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052d4b expref: -99 pid: 66702 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764853.166616:0:66980:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c207 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d4b expref: -99 pid: 66702 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764853.166717:0:66980:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764853.166727:0:66980:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764853.166736:0:66980:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764853.166744:0:66980:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:1.0:1775764853.166750:0:66980:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764853.166754:0:66980:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764853.166757:0:66980:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c1d6 00000080:00010000:1.0:1775764853.166761:0:66980:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c1d6 from it ffffa60b01e6bbb0 00010000:00010000:1.0:1775764853.166764:0:66980:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764853.166771:0:66980:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c1d6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052c9c expref: -99 pid: 66196 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:1.0:1775764853.166780:0:66980:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764853.166787:0:66980:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 100000 00010000:00010000:1.0:1775764853.196215:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1775764853.196229:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c20e lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03052d67 expref: -99 pid: 66981 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.196242:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5e600 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:1.0:1775764853.196247:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c20e lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03052d67 expref: -99 pid: 66981 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.196255:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c20e lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03052d67 expref: -99 pid: 66981 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.196265:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c20e lrc: 3/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052d67 expref: -99 pid: 66981 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.196295:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e000/0x73cfd86cfc17baf9 lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305144d expref: -99 pid: 61408 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1775764853.196350:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c20e lrc: 2/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052d67 expref: -99 pid: 66981 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.196357:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764853.196389:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c20e lrc: 1/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052d67 expref: -99 pid: 66981 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.196396:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c20e lrc: 0/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052d67 expref: -99 pid: 66981 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.196422:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764853.196424:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764853.196425:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764853.196728:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098698880:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764853.214292:0:2376:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098698880:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764853.330055:0:66748:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:66748:1862022098699136:192.168.202.128@tcp:101:bash.0 00010000:00010000:2.0:1775764853.330082:0:66748:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.330110:0:66748:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.330118:0:66748:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.330135:0:66748:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.330145:0:66748:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.330158:0:66748:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.330165:0:66748:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.330170:0:66748:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.330181:0:66748:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8b2382dac700 x1862022098699136/t34359739361(34359739361) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764853.330193:0:66748:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382dac700 x1862022098699136/t34359739361(34359739361) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764853.330204:0:66748:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764853.330215:0:66748:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.330226:0:66748:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/27 (0 0) 00000080:00200000:2.0:1775764853.330232:0:66748:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764853.330246:0:66748:0:(namei.c:1620:ll_create_it()) VFS Op:name=f27.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000080:00200000:2.0:1775764853.330270:0:66748:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) 00000080:00200000:2.0:1775764853.330439:0:66748:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1b:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1775764853.330506:0:66748:0:(namei.c:1586:ll_create_node()) setting l_ast_data to inode [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) 00000080:00010000:2.0:1775764853.330510:0:66748:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c215 00000080:00010000:2.0:1775764853.330520:0:66748:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) 00000080:00200008:2.0:1775764853.330535:0:66748:0:(file.c:1057:ll_file_open()) START file f27.sanity-pcc:[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), flags 101101 00000002:00100000:2.0:1775764853.330554:0:66748:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382dac700 x1862022098699136/t34359739361(34359739361) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764853.330589:0:66748:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c215 from it ffff8b23891e8720 00010000:00010000:2.0:1775764853.330593:0:66748:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.330602:0:66748:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764853.330657:0:66748:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), inode mode 81a4 mask 40 00000080:00200008:2.0:1775764853.330739:0:66748:0:(file.c:2612:do_file_write_iter()) START file f27.sanity-pcc:[0x200003ab4:0x1b:0x0], ppos: 0, count: 22 00000080:00200000:2.0:1775764853.330752:0:66748:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 22 bytes 00000080:00200000:2.0:1775764853.330758:0:66748:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 22 00000080:00200000:2.0:1775764853.330770:0:66748:0:(file.c:2058:ll_file_io_generic()) f27.sanity-pcc: write ppos: 0, bytes: 22 00000080:00200000:2.0:1775764853.330774:0:66748:0:(file.c:1815:ll_io_set_mirror()) f27.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764853.330783:0:66748:0:(cl_io.c:214:cl_io_rw_init()) header@00000000cf6c252a[0x110, 1, [0x200003ab4:0x1b:0x0] hash] 00000020:00200000:2.0:1775764853.330785:0:66748:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 22) 0 0 00000080:00200000:2.0:1775764853.330789:0:66748:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1b:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764853.330803:0:66748:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:2.0:1775764853.330810:0:66748:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 22) 22, 22 00020000:00200000:2.0:1775764853.330814:0:66748:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764853.330841:0:66748:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1775764853.330854:0:66748:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764853.330858:0:66748:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8030, osc lock ffff8b238a3f8be0, flags 0x0 00010000:00010000:2.0:1775764853.330899:0:66748:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 3/0,1 mode: --/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.330909:0:66748:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 3/0,1 mode: --/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.330919:0:66748:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 3/0,1 mode: --/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764853.330940:0:66748:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:66748:1862022098699392:192.168.202.128@tcp:101:bash.0 00000100:00100000:2.0:1775764853.331001:0:66748:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bb40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.335737:0:66748:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:66748:1862022098699392:192.168.202.128@tcp:101:bash.0 00010000:00010000:2.0:1775764853.335765:0:66748:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 4/0,1 mode: --/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.335781:0:66748:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.335793:0:66748:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.335807:0:66748:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1775764853.335820:0:66748:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.335832:0:66748:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.335850:0:66748:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764853.335879:0:66748:0:(vvp_io.c:1286:vvp_io_write_start()) f27.sanity-pcc: write [0, 22) append 0 append lockpos 0 00000080:00200000:2.0:1775764853.335897:0:66748:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 22 bytes 00000080:00200000:2.0:1775764853.335953:0:66748:0:(rw26.c:858:ll_write_end()) pos 0, len 22, copied 22 00000080:00200000:2.0:1775764853.335964:0:66748:0:(rw26.c:916:ll_write_end()) page@0000000026446a92[3 00000000d4073652 2 1 0000000077ca0792] 00000080:00200000:2.0:1775764853.335967:0:66748:0:(rw26.c:916:ll_write_end()) vmpage @00000000fd54f72b 17ffffc0001001 3:0 ffff8b23baeae8c0 0 lru 00000080:00200000:2.0:1775764853.335974:0:66748:0:(rw26.c:916:ll_write_end()) osc-page@000000006006ce72 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 000000003596ffcd 00000000ed923912 > 3< 0 0 > 4< 0 0 8 18743296 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764853.335975:0:66748:0:(rw26.c:916:ll_write_end()) end page@0000000026446a92 00000080:00200000:2.0:1775764853.335976:0:66748:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764853.335981:0:66748:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 22 prio 0 00000008:00010000:2.0:1775764853.336002:0:66748:0:(osc_cache.c:848:osc_extent_find()) ### extent: 0000000033f75963 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764853.336022:0:66748:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2390ec5d38, count 1, dirtied 1 00000080:00200000:2.0:1775764853.336027:0:66748:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 22 bytes, tot: 22 00000080:00200000:2.0:1775764853.336029:0:66748:0:(vvp_io.c:1353:vvp_io_write_start()) f27.sanity-pcc: write bytes 0, result: 22 00000020:00010000:2.0:1775764853.336047:0:66748:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b8a8 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764853.336050:0:66748:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b8a8 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764853.336053:0:66748:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8030 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764853.336057:0:66748:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.336067:0:66748:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1775764853.336078:0:66748:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b8a8 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764853.336079:0:66748:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8030 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764853.336085:0:66748:0:(vvp_io.c:517:vvp_io_advance()) advancing 22 bytes 00000080:00200000:2.0:1775764853.336090:0:66748:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:2.0:1775764853.336114:0:66748:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1b:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764853.336117:0:66748:0:(file.c:2206:ll_file_io_generic()) f27.sanity-pcc: 2 io complete with rc: 0, result: 22, restart: 0 00000080:00200000:2.0:1775764853.336124:0:66748:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1b:0x0] updating i_size 22 i_blocks 1 00000080:00200000:2.0:1775764853.336129:0:66748:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 22 00000080:00200008:2.0:1775764853.336177:0:66748:0:(file.c:447:ll_file_release()) START file f27.sanity-pcc:[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), flags 100001 00000002:00100000:2.0:1775764853.336228:0:66748:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b2382dac700 x1862022098699136/t34359739361(34359739361) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764869 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764853.336236:0:66748:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764853.336240:0:66748:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6680 x1862022098699648/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764853.336257:0:66748:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:66748:1862022098699648:192.168.202.128@tcp:35:bash.0 00000100:00100000:2.0:1775764853.336322:0:66748:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bb40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.339367:0:66748:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:66748:1862022098699648:192.168.202.128@tcp:35:bash.0 00000002:00100000:2.0:1775764853.339403:0:66748:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b2382dac700 x1862022098699136/t34359739361(34359739361) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1775764853.381765:0:66995:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764853.381779:0:66995:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764853.381802:0:66995:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764853.381867:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66995 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.381875:0:66995:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66995 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.381883:0:66995:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66995 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764853.381903:0:66995:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66995:1862022098699776:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764853.382019:0:66995:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654540 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.384555:0:66995:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66995:1862022098699776:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764853.384588:0:66995:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.384602:0:66995:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.384616:0:66995:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.384636:0:66995:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.384646:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.384652:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.384657:0:66995:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.384664:0:66995:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864fd880 x1862022098699776/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764853.384674:0:66995:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.384683:0:66995:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764853.384688:0:66995:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764853.384715:0:66995:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764853.384721:0:66995:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c223 00000080:00010000:2.0:1775764853.384726:0:66995:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764853.384729:0:66995:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c223 from it ffffa60b01eb3cb8 00010000:00010000:2.0:1775764853.384731:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.384739:0:66995:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764853.384800:0:66995:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764853.384812:0:66995:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764853.384819:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.384826:0:66995:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764853.384830:0:66995:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764853.384835:0:66995:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764853.384837:0:66995:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c223 00000080:00010000:2.0:1775764853.384839:0:66995:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c223 from it ffffa60b01eb3b58 00010000:00010000:2.0:1775764853.384841:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.384845:0:66995:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764853.384850:0:66995:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764853.384856:0:66995:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f27.sanity-pcc, flags=8 00000080:00200008:2.0:1775764853.384862:0:66995:0:(file.c:6226:ll_getattr_dentry()) START file f27.sanity-pcc:[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764853.384865:0:66995:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88),name=f27.sanity-pcc 00000002:00010000:2.0:1775764853.384873:0:66995:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1b:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764853.384878:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.384883:0:66995:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764853.384886:0:66995:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764853.384888:0:66995:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) 00000080:00010000:2.0:1775764853.384890:0:66995:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c215 00000080:00010000:2.0:1775764853.384892:0:66995:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c215 from it ffffa60b01eb3cb8 00010000:00010000:2.0:1775764853.384893:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.384897:0:66995:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764853.384958:0:66995:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1b:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1775764853.384965:0:66995:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x1b:0x0] 00000008:00010000:2.0:1775764853.384985:0:66995:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764853.384988:0:66995:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8e30, osc lock ffff8b238a3f8be0, flags 0x400001002 00010000:00010000:2.0:1775764853.384999:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.385006:0:66995:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1775764853.385015:0:66995:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764853.385017:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.385022:0:66995:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 5/0,0 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1775764853.385028:0:66995:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764853.385034:0:66995:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1b:0x0] updating i_size 22 i_blocks 1 00000020:00010000:2.0:1775764853.385036:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027dda8 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764853.385038:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027dda8 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764853.385040:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8e30 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764853.385042:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027dda8 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764853.385044:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8e30 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764853.385055:0:66995:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1b:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764853.385079:0:66995:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764853.385084:0:66995:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764853.385087:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.385092:0:66995:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764853.385096:0:66995:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764853.385098:0:66995:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764853.385100:0:66995:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c223 00000080:00010000:2.0:1775764853.385102:0:66995:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c223 from it ffffa60b01eb3bf8 00010000:00010000:2.0:1775764853.385104:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.385108:0:66995:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764853.385112:0:66995:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764853.385115:0:66995:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f27.sanity-pcc, flags=265 00000080:00200000:2.0:1775764853.385117:0:66995:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), inode mode 81a4 mask 46 00000080:00200008:2.0:1775764853.385124:0:66995:0:(file.c:1057:ll_file_open()) START file f27.sanity-pcc:[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), flags 104002 00000002:00010000:2.0:1775764853.385135:0:66995:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1b:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240104003 00010000:00010000:2.0:1775764853.385178:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c22a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66995 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.385185:0:66995:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c22a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66995 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764853.385196:0:66995:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764853.385201:0:66995:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7480 x1862022098700032/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764853.385208:0:66995:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c22a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66995 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764853.385224:0:66995:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66995:1862022098700032:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764853.385353:0:66995:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bb40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764853.388322:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764853.389351:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764853.389358:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.389383:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2381497400 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764853.389388:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.389396:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.389405:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764853.389497:0:46150:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1b:0x0]: losing layout lock 00010000:00010000:2.0:1775764853.389564:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.389572:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764853.389595:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.389603:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c215 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052d8a expref: -99 pid: 66748 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.389882:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764853.389884:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764853.389886:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764853.398953:0:66995:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66995:1862022098700032:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764853.398971:0:66995:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CW ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c22a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052db4 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.398994:0:66995:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c22a lrc: 4/1,0 mode: --/CW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052db4 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.398999:0:66995:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c22a lrc: 4/1,0 mode: --/CW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052db4 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.399008:0:66995:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c22a lrc: 4/1,0 mode: CW/CW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052db4 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.399015:0:66995:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c22a lrc: 4/1,0 mode: CW/CW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052db4 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.399026:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c22a lrc: 5/1,1 mode: CW/CW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052db4 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.399033:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c22a lrc: 5/1,1 mode: CW/CW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052db4 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.399039:0:66995:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c22a lrc: 4/0,1 mode: CW/CW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052db4 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.399050:0:66995:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23b9ea7480 x1862022098700032/t34359739363(34359739363) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764853.399064:0:66995:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7480 x1862022098700032/t34359739363(34359739363) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764853.399079:0:66995:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c22a lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052db4 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.399090:0:66995:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 8 res 8589949620/27 (0 0) 00000080:00200000:2.0:1775764853.399097:0:66995:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764853.399187:0:66995:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c22a 00000080:00010000:2.0:1775764853.399195:0:66995:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c22a from it ffffa60b01eb3bf0 00010000:00010000:2.0:1775764853.399198:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c22a lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052db4 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.399207:0:66995:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c22a lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052db4 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:2.0:1775764853.399233:0:66995:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7480 x1862022098700032/t34359739363(34359739363) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764853.399592:0:66995:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) cmd=c00c66f3 arg=7ffd5999abf4 00000002:00010000:2.0:1775764853.399615:0:66995:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1b:0x0]) in obj [0x200003ab4:0x1b:0x0], intent: open flags 01004040000002 00010000:00010000:2.0:1775764853.399713:0:66995:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c22a lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052db4 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.399846:0:66995:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c22a lrc: 1/0,0 mode: --/CW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052db4 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.399854:0:66995:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764853.399857:0:66995:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030a00/0x73cfd86cfc17c22a lrc: 0/0,0 mode: --/CW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052db4 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.399893:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c231 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66995 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.399905:0:66995:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x81000000001000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c231 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66995 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764853.399915:0:66995:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764853.399922:0:66995:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4380 x1862022098700416/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764853.399931:0:66995:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c231 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x81000000000000 pid: 66995 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764853.399952:0:66995:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66995:1862022098700416:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764853.400178:0:66995:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bb40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.408622:0:66995:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66995:1862022098700416:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764853.408644:0:66995:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PW ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c231 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e03052dc9 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.408654:0:66995:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c231 lrc: 4/1,0 mode: --/PW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e03052dc9 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.408663:0:66995:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c231 lrc: 4/1,0 mode: PW/PW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e03052dc9 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.408670:0:66995:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c231 lrc: 4/1,0 mode: PW/PW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e03052dc9 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.408681:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c231 lrc: 5/1,1 mode: PW/PW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e03052dc9 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.408694:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c231 lrc: 5/1,1 mode: PW/PW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e03052dc9 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.408701:0:66995:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c231 lrc: 4/0,1 mode: PW/PW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e03052dc9 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.408711:0:66995:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=600002b, status=0 req@ffff8b23b9ea4380 x1862022098700416/t34359739364(34359739364) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764853.408724:0:66995:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4380 x1862022098700416/t34359739364(34359739364) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764853.408739:0:66995:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c231 lrc: 3/0,1 mode: PW/PW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e03052dc9 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.408879:0:66995:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 2 res 8589949620/27 (0 0) 00000080:00200000:2.0:1775764853.408887:0:66995:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab4:0x1b:0x0] suppgids 0 0: rc 0 00000002:00100000:2.0:1775764853.408905:0:66995:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4380 x1862022098700416/t34359739364(34359739364) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764869 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764853.408914:0:66995:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c231 00000080:00010000:2.0:1775764853.408920:0:66995:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c231 from it ffffa60b01eb3cc0 00010000:00010000:2.0:1775764853.408923:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c231 lrc: 3/0,1 mode: PW/PW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e03052dc9 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.408931:0:66995:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c231 lrc: 2/0,0 mode: PW/PW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x4/0x0 rrc: 2 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e03052dc9 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200080:2.0:1775764853.409113:0:66995:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) cmd=c00c66f3 arg=11192a0 00000080:00200008:2.0:1775764853.411000:0:66995:0:(file.c:2454:do_file_read_iter()) START file f27.sanity-pcc:[0x200003ab4:0x1b:0x0], ppos: 0, count: 1048576 00000080:00200000:2.0:1775764853.411042:0:66995:0:(file.c:2058:ll_file_io_generic()) f27.sanity-pcc: read ppos: 0, bytes: 1048576 00000080:00200000:2.0:1775764853.411048:0:66995:0:(file.c:1815:ll_io_set_mirror()) f27.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764853.411055:0:66995:0:(cl_io.c:214:cl_io_rw_init()) header@00000000cf6c252a[0x110, 2, [0x200003ab4:0x1b:0x0] hash] 00000020:00200000:2.0:1775764853.411057:0:66995:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 1048576) 2048 0 00000080:00200000:2.0:1775764853.411062:0:66995:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1b:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00010000:00010000:2.0:1775764853.411081:0:66995:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 31 res 8589949620/27 (0 0) 00000080:00010000:2.0:1775764853.411092:0:66995:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) 00000002:00010000:2.0:1775764853.411103:0:66995:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1b:0x0]) in obj [0x200003ab4:0x1b:0x0], intent: layout flags 0 00010000:00010000:2.0:1775764853.411151:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66995 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.411160:0:66995:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66995 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.411169:0:66995:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66995 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764853.411192:0:66995:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66995:1862022098700544:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764853.411573:0:66995:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bb40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.414239:0:66995:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66995:1862022098700544:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764853.414274:0:66995:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.414289:0:66995:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.414321:0:66995:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea6300 x1862022098700544/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/400 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764853.414332:0:66995:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764853.414343:0:66995:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.414355:0:66995:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/27 (0 0) 00000080:00200000:2.0:1775764853.414360:0:66995:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x1b:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764853.414392:0:66995:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c238 00000080:00010000:2.0:1775764853.414401:0:66995:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c238 from it ffffa60b01eb3778 00010000:00010000:2.0:1775764853.414404:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.414423:0:66995:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.414446:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.414486:0:66995:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764853.414494:0:66995:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1b:0x0](000000000e8c9725) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764853.414567:0:66995:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1b:0x0]: layout version change: 4294967294 -> 0 00010000:00010000:2.0:1775764853.414590:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.414596:0:66995:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00020000:00200000:2.0:1775764853.414617:0:66995:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:2.0:1775764853.414621:0:66995:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764853.414647:0:66995:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:2.0:1775764853.414661:0:66995:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 1 00000008:00010000:2.0:1775764853.414665:0:66995:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8e30, osc lock ffff8b238a3f8be0, flags 0x40000 00010000:00010000:2.0:1775764853.414684:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.414694:0:66995:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.414707:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.414715:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.414722:0:66995:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764853.414736:0:66995:0:(vvp_io.c:826:vvp_io_read_start()) f27.sanity-pcc: read [0, 1048576) 00000001:00010000:2.0:1775764853.414749:0:66995:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x1b:0x0] 00000008:00010000:2.0:1775764853.414765:0:66995:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764853.414767:0:66995:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8030, osc lock ffff8b238a3f8130, flags 0x400001002 00010000:00010000:2.0:1775764853.414774:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.414782:0:66995:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1775764853.414790:0:66995:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764853.414794:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 8/1,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.414801:0:66995:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 7/1,0 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1775764853.414807:0:66995:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764853.414813:0:66995:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1b:0x0] updating i_size 22 i_blocks 1 00000020:00010000:2.0:1775764853.414818:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027dda8 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764853.414821:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027dda8 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764853.414824:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8030 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764853.414827:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027dda8 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764853.414830:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8030 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764853.414838:0:66995:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:0 00000020:00010000:2.0:1775764853.414852:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b8a8 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764853.414854:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b8a8 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764853.414857:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8e30 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764853.414860:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.414868:0:66995:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00010000:2.0:1775764853.414877:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b8a8 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764853.414879:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8e30 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764853.414884:0:66995:0:(vvp_io.c:517:vvp_io_advance()) advancing 22 bytes 00000080:00200000:2.0:1775764853.414901:0:66995:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1b:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764853.414905:0:66995:0:(file.c:2206:ll_file_io_generic()) f27.sanity-pcc: 1 io complete with rc: 0, result: 22, restart: 0 00000080:00200000:2.0:1775764853.414910:0:66995:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 22 00000080:00200008:2.0:1775764853.415028:0:66995:0:(file.c:2454:do_file_read_iter()) START file f27.sanity-pcc:[0x200003ab4:0x1b:0x0], ppos: 22, count: 1048576 00000080:00200000:2.0:1775764853.415047:0:66995:0:(file.c:2058:ll_file_io_generic()) f27.sanity-pcc: read ppos: 22, bytes: 1048576 00000080:00200000:2.0:1775764853.415051:0:66995:0:(file.c:1815:ll_io_set_mirror()) f27.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764853.415057:0:66995:0:(cl_io.c:214:cl_io_rw_init()) header@00000000cf6c252a[0x110, 2, [0x200003ab4:0x1b:0x0] hash] 00000020:00200000:2.0:1775764853.415058:0:66995:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [22, 1048598) 2048 0 00000080:00200000:2.0:1775764853.415062:0:66995:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1b:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764853.415070:0:66995:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 22 chunk: [22, 1048598) 1048598, 1048576 00020000:00200000:2.0:1775764853.415072:0:66995:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764853.415086:0:66995:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 256] 00000008:00010000:2.0:1775764853.415092:0:66995:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 1 00000008:00010000:2.0:1775764853.415094:0:66995:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8e30, osc lock ffff8b238a3f8be0, flags 0x40000 00010000:00010000:2.0:1775764853.415102:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764853.415111:0:66995:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1052671) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764853.415676:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764853.415687:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764853.415693:0:66995:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764853.415703:0:66995:0:(vvp_io.c:826:vvp_io_read_start()) f27.sanity-pcc: read [22, 1048598) 00000001:00010000:2.0:1775764853.415713:0:66995:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x1b:0x0] 00000008:00010000:2.0:1775764853.415720:0:66995:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764853.415722:0:66995:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8030, osc lock ffff8b238a3f8130, flags 0x400001002 00010000:00010000:2.0:1775764853.415730:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764853.415740:0:66995:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:2.0:1775764853.415826:0:66995:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764853.415832:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 8/1,1 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764853.415839:0:66995:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 7/1,0 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:2.0:1775764853.415851:0:66995:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764853.415857:0:66995:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1b:0x0] updating i_size 22 i_blocks 1 00000020:00010000:2.0:1775764853.415860:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027dda8 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764853.415862:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027dda8 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764853.415864:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8030 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764853.415866:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027dda8 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764853.415868:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8030 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764853.415874:0:66995:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:22 00000020:00010000:2.0:1775764853.415882:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b8a8 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764853.415884:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b8a8 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764853.415886:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8e30 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764853.415889:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764853.415897:0:66995:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:2.0:1775764853.415907:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b8a8 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764853.415909:0:66995:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8e30 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764853.415913:0:66995:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1775764853.415932:0:66995:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1b:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764853.415935:0:66995:0:(file.c:2206:ll_file_io_generic()) f27.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1775764853.415937:0:66995:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200000:2.0:1775764853.416105:0:66995:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1b:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764853.416114:0:66995:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:1.0:1775764853.419437:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119081088 00000100:00100000:1.0:1775764853.419447:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119081088 00000100:00100000:1.0:1775764853.419527:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 295 00000100:00100000:1.0:1775764853.419534:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dad180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42946:x1862022119081088:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764853.419550:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b2389398000 lock: 0000000090955e3d/0x73cfd86cfc17c21c lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e03052d98 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764853.419629:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764853.419637:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dad180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42946:x1862022119081088:12345-192.168.202.128@tcp:104: Request processed in 102us (293us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764853.419644:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 295 00000100:00100000:1.0:1775764853.429309:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098700928:192.168.202.128@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764853.432928:0:66995:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c231 lrc: 2/0,0 mode: PW/PW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x57b8ff0e03052dc9 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.432948:0:66995:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c231 lrc: 0/0,0 mode: --/PW res: [0x200003ab4:0x1b:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x57b8ff0e03052dc9 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.432968:0:66995:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea4380 x1862022098700416/t34359739364(34359739364) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764869 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764853.432980:0:66995:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764853.432985:0:66995:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6a00 x1862022098701184/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 504/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764853.433007:0:66995:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66995:1862022098701184:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764853.433079:0:66995:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bb40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764853.438020:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764853.438785:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764853.438793:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.438802:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2380030000 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764853.438807:0:46150:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.438815:0:46150:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.438822:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 3/0,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764853.438885:0:46150:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1b:0x0]: losing layout lock 00010000:00010000:2.0:1775764853.438950:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 2/0,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.438958:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764853.438989:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.438997:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c238 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03052dd7 expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.439019:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764853.439021:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764853.439023:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764853.439050:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764853.439065:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098701312:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764853.445016:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098701312:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764853.448803:0:66995:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66995:1862022098701184:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764853.448838:0:66995:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea4380 x1862022098700416/t34359739364(34359739364) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764853.448913:0:66995:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) 00000002:00010000:2.0:1775764853.448924:0:66995:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1b:0x0]) in obj [0x200003ab4:0x1b:0x0], intent: layout flags 0 00010000:00010000:2.0:1775764853.448972:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66995 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.448983:0:66995:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66995 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.448992:0:66995:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66995 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764853.449011:0:66995:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66995:1862022098701440:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764853.449091:0:66995:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bb40 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.452028:0:66995:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66995:1862022098701440:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764853.452058:0:66995:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.452072:0:66995:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764853.452087:0:66995:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea6a00 x1862022098701440/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/400 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764853.452097:0:66995:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764853.452109:0:66995:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.452121:0:66995:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/27 (0 0) 00000080:00200000:2.0:1775764853.452126:0:66995:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x1b:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764853.452144:0:66995:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c23f 00000080:00010000:2.0:1775764853.452152:0:66995:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c23f from it ffffa60b01eb3b70 00010000:00010000:2.0:1775764853.452156:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764853.452166:0:66995:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.452188:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.452198:0:66995:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764853.452207:0:66995:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1b:0x0](000000000e8c9725) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.452281:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764853.452289:0:66995:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764853.452332:0:66995:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764853.452343:0:66995:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764853.452385:0:66995:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764853.452415:0:66995:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.452424:0:66995:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764853.452431:0:66995:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1b:0x0](000000000e8c9725) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764853.452623:0:66995:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764853.452629:0:66995:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764853.452658:0:66995:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764853.452702:0:66995:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1b:0x0]: layout version change: 4294967294 -> 1 00010000:00010000:2.0:1775764853.452714:0:66995:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764853.452721:0:66995:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:2.0:1775764853.452967:0:66995:0:(file.c:447:ll_file_release()) START file f27.sanity-pcc:[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), flags 104002 00000002:00100000:2.0:1775764853.453015:0:66995:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea7480 x1862022098700032/t34359739363(34359739363) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764869 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764853.453026:0:66995:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764853.453031:0:66995:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6a00 x1862022098701696/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764853.453054:0:66995:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66995:1862022098701696:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764853.453185:0:66995:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b0c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764853.456588:0:66995:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:66995:1862022098701696:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764853.456620:0:66995:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea7480 x1862022098700032/t34359739363(34359739363) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764853.545395:0:67005:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764853.545435:0:67005:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764853.545711:0:67005:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764853.545730:0:67005:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764853.545740:0:67005:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764853.545748:0:67005:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764853.545839:0:67005:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c223 00000080:00010000:1.0:1775764853.545848:0:67005:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c223 from it ffffa60b01eabbb0 00010000:00010000:1.0:1775764853.545852:0:67005:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764853.545861:0:67005:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764853.545877:0:67005:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764853.545886:0:67005:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:1.0:1775764853.545914:0:67005:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7fff87ad5600 00000100:00100000:1.0:1775764853.545982:0:67005:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67005:1862022098701824:192.168.202.128@tcp:34:lfs.0 00000100:00100000:1.0:1775764853.546127:0:67005:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654cc0 going to sleep for 11 seconds 00000080:00200000:1.0:1775764853.961655:0:67021:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764853.961687:0:67021:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764853.961707:0:67021:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764853.961723:0:67021:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764853.961733:0:67021:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764853.961741:0:67021:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764853.961745:0:67021:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c223 00000080:00010000:1.0:1775764853.961752:0:67021:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c223 from it ffffa60b01eabbf8 00010000:00010000:1.0:1775764853.961755:0:67021:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764853.961765:0:67021:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764853.961782:0:67021:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764853.961797:0:67021:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764853.961804:0:67021:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764853.961810:0:67021:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764853.961817:0:67021:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764853.961823:0:67021:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764853.961827:0:67021:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764853.961830:0:67021:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c223 00000080:00010000:1.0:1775764853.961833:0:67021:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c223 from it ffffa60b01eabb38 00010000:00010000:1.0:1775764853.961836:0:67021:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764853.961843:0:67021:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764853.961850:0:67021:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:1.0:1775764853.961855:0:67021:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f27.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b238bc71c80, open_flags 8042, mode 81a4 opened 0 00000080:00200000:1.0:1775764853.961865:0:67021:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f27.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000002:00010000:1.0:1775764853.961885:0:67021:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f27.sanity-pcc,[0x200003ab4:0x1d:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0100103 00010000:00010000:1.0:1775764853.961897:0:67021:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764853.961950:0:67021:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764853.961958:0:67021:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764853.961960:0:67021:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741800/0x73cfd86cfc17c223 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052da6 expref: -99 pid: 66995 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764853.961992:0:67021:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c246 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67021 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764853.962004:0:67021:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c246 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67021 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764853.962016:0:67021:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764853.962029:0:67021:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd180 x1862022098702080/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764853.962039:0:67021:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c246 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67021 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764853.962063:0:67021:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:67021:1862022098702080:192.168.202.128@tcp:101:multiop.0 00000100:00100000:1.0:1775764853.962324:0:67021:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654240 going to sleep for 11 seconds 00010000:00010000:2.0:1775764854.712126:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764856.759625:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1775764856.896185:0:67048:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764856.896207:0:67048:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764856.898772:0:67048:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000080:00200000:2.0:1775764857.395012:0:67075:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764857.395133:0:67075:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764857.395162:0:67075:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764857.395182:0:67075:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764857.395193:0:67075:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764857.395203:0:67075:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764857.395207:0:67075:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c24d 00000080:00010000:2.0:1775764857.395215:0:67075:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c24d from it ffffa60b01eb3bb0 00010000:00010000:2.0:1775764857.395220:0:67075:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764857.395231:0:67075:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764857.395250:0:67075:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764857.395336:0:67075:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764857.395436:0:67075:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffce0ed4a90 00000080:00200008:2.0:1775764857.395579:0:67075:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200008:1.0:1775764857.501622:0:67081:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764857.501639:0:67081:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764857.501668:0:67081:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764857.501691:0:67081:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764857.501708:0:67081:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764857.501716:0:67081:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764857.501727:0:67081:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764857.501731:0:67081:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c24d 00000080:00010000:1.0:1775764857.501740:0:67081:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c24d from it ffffa60b01f0bcb8 00010000:00010000:1.0:1775764857.501745:0:67081:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764857.501755:0:67081:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764857.501790:0:67081:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764857.501798:0:67081:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764857.501803:0:67081:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764857.501812:0:67081:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764857.501818:0:67081:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764857.501822:0:67081:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764857.501824:0:67081:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c24d 00000080:00010000:1.0:1775764857.501828:0:67081:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c24d from it ffffa60b01f0bb58 00010000:00010000:1.0:1775764857.501831:0:67081:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1775764857.501838:0:67081:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:1.0:1775764857.501846:0:67081:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764857.501854:0:67081:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f27.sanity-pcc, flags=8 00000080:00200008:1.0:1775764857.501861:0:67081:0:(file.c:6226:ll_getattr_dentry()) START file f27.sanity-pcc:[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764857.501864:0:67081:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88),name=f27.sanity-pcc 00000002:00010000:1.0:1775764857.501877:0:67081:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1b:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764857.501885:0:67081:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c246 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052e1d expref: -99 pid: 67021 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764857.501893:0:67081:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c246 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052e1d expref: -99 pid: 67021 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764857.501898:0:67081:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764857.501902:0:67081:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) 00000080:00010000:1.0:1775764857.501904:0:67081:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c246 00000080:00010000:1.0:1775764857.501907:0:67081:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c246 from it ffffa60b01f0bcb8 00010000:00010000:1.0:1775764857.501910:0:67081:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c246 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052e1d expref: -99 pid: 67021 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764857.501916:0:67081:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c246 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052e1d expref: -99 pid: 67021 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764857.501950:0:67081:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764857.501957:0:67081:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764857.501963:0:67081:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764857.501971:0:67081:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:1.0:1775764857.501977:0:67081:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764857.501980:0:67081:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764857.501983:0:67081:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c24d 00000080:00010000:1.0:1775764857.501987:0:67081:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c24d from it ffffa60b01f0bbf8 00010000:00010000:1.0:1775764857.501990:0:67081:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1775764857.501997:0:67081:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:1.0:1775764857.502005:0:67081:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764857.502009:0:67081:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f27.sanity-pcc, flags=265 00000080:00200000:1.0:1775764857.502012:0:67081:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), inode mode 81a4 mask 46 00000080:00200008:1.0:1775764857.502019:0:67081:0:(file.c:1057:ll_file_open()) START file f27.sanity-pcc:[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), flags 154402 00000080:00200080:1.0:1775764857.502053:0:67081:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) cmd=c00466fc arg=7ffdf356a1ac 00800000:00000080:1.0:1775764857.502071:0:67081:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d9 len=56 karg=ffff8b23891edc80 uarg=0000000000000000 00000002:00000080:1.0:1775764857.502079:0:67081:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d9 len=56 karg=ffff8b23891edc80 uarg=0000000000000000 00010000:00010000:1.0:1775764857.502116:0:67081:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c246 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052e1d expref: -99 pid: 67021 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764857.505873:0:67081:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1b:0x0]: losing layout lock 00010000:00010000:1.0:1775764857.505944:0:67081:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c254 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052e47 expref: -99 pid: 67065 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000002:00100000:1.0:1775764857.506042:0:67081:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864f1f80 x1862022098702464/t34359739376(34359739376) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764873 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cat.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764857.506058:0:67081:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764857.506064:0:67081:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fce00 x1862022098702848/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764857.506094:0:67081:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67081:1862022098702848:192.168.202.128@tcp:35:lfs.0 00000100:00100000:1.0:1775764857.506405:0:67081:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764857.518409:0:67081:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67081:1862022098702848:192.168.202.128@tcp:35:lfs.0 00000002:00100000:2.0:1775764857.518489:0:67081:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864f1f80 x1862022098702464/t34359739376(34359739376) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764873 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cat.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764857.518566:0:67081:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1b:0x0]: losing layout lock 00010000:00010000:2.0:1775764857.518627:0:67081:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764857.518647:0:67081:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1b:0x0]: losing layout lock 00010000:00010000:2.0:1775764857.518683:0:67081:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c246 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052e1d expref: -99 pid: 67021 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764857.518688:0:67081:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c254 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052e47 expref: -99 pid: 67065 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764857.518693:0:67081:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764857.518696:0:67081:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00000100:00100000:2.0:1775764857.518713:0:67081:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67081:1862022098702976:192.168.202.128@tcp:103:lfs.0 00000100:00100000:2.0:1775764857.518800:0:67081:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bf00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764857.526450:0:67081:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67081:1862022098702976:192.168.202.128@tcp:103:lfs.0 00010000:00010000:2.0:1775764857.526544:0:67081:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c246 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052e1d expref: -99 pid: 67021 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764857.526565:0:67081:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c254 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052e47 expref: -99 pid: 67065 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764857.526574:0:67081:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c23f lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052dec expref: -99 pid: 66995 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000020:00100000:2.0:1775764857.526678:0:67081:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 58, max 7 00000100:00100000:2.0:1775764857.526695:0:67081:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fe680 x1862022098703104/t0(0) o58->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764857.526739:0:67081:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67081:1862022098703104:192.168.202.128@tcp:58:lfs.0 00000100:00100000:2.0:1775764857.526870:0:67081:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bf00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764857.533902:0:67081:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67081:1862022098703104:192.168.202.128@tcp:58:lfs.0 00000080:00010000:2.0:1775764857.533975:0:67081:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) 00000002:00010000:2.0:1775764857.533989:0:67081:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1b:0x0]) in obj [0x200003ab4:0x1b:0x0], intent: layout flags 0 00010000:00010000:2.0:1775764857.534054:0:67081:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67081 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764857.534064:0:67081:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67081 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764857.534075:0:67081:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67081 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764857.534100:0:67081:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67081:1862022098703232:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764857.534204:0:67081:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bf00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764857.538561:0:67081:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67081:1862022098703232:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764857.538607:0:67081:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764857.538623:0:67081:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764858.197595:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119082752 00000100:00100000:1.0:1775764858.197608:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119082752 00000100:00100000:1.0:1775764858.197685:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 297 00000100:00100000:1.0:1775764858.197699:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dac000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42356:x1862022119082752:12345-192.168.202.128@tcp:107:hsm_cdtr.0 00000100:00100000:1.0:1775764858.197834:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dac000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42356:x1862022119082752:12345-192.168.202.128@tcp:107:hsm_cdtr.0 Request processed in 138us (470us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764858.197842:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 297 00000080:00200000:2.0:1775764858.258121:0:67083:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0x1b:0x0], updating i_size 22 00000080:00200000:2.0:1775764858.258134:0:67083:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88): rc = 0 00000080:00010000:2.0:1775764858.258145:0:67083:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) for lock 0x0 00000080:00200008:2.0:1775764858.258179:0:67083:0:(file.c:1057:ll_file_open()) START file 0x200003ab4:0x1b:0x0:[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), flags 1504000 00000002:00100000:2.0:1775764858.258223:0:67083:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864f0e00 x1862022098704256/t34359739378(34359739378) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764874 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764858.258296:0:67083:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), inode mode 81a4 mask 44 00000080:00200080:2.0:1775764858.258338:0:67083:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086696 arg=7fdbe8000ce0 00800000:00000080:2.0:1775764858.258358:0:67083:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=c0086696 len=4128 karg=ffff8b2391290000 uarg=0000000000000000 00000002:00000080:2.0:1775764858.258376:0:67083:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=c0086696 len=4128 karg=ffff8b2391290000 uarg=0000000000000000 00000002:00000080:2.0:1775764858.258382:0:67083:0:(mdc_request.c:1738:mdc_ioc_fid2path()) path get [0x200003ab4:0x1b:0x0] from 18446744073709551615 #0 00000100:00100000:2.0:1775764858.258437:0:67083:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098704384:192.168.202.128@tcp:53:lhsmtool_posix.0 00000100:00100000:2.0:1775764858.258675:0:67083:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bf00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764858.277312:0:67083:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098704384:192.168.202.128@tcp:53:lhsmtool_posix.0 00000002:00000080:2.0:1775764858.277357:0:67083:0:(mdc_request.c:1758:mdc_ioc_fid2path()) path got [0x200000007:0x1:0x0] from 18446744073709551615 #0: f27.sanity-pcc 00000080:00200000:2.0:1775764858.277521:0:67083:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764858.277550:0:67083:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764858.277824:0:67083:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:2.0:1775764858.277843:0:67083:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:2.0:1775764858.277853:0:67083:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764858.277862:0:67083:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764858.277867:0:67083:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c24d 00000080:00010000:2.0:1775764858.277875:0:67083:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c24d from it ffffa60b0200fbf8 00010000:00010000:2.0:1775764858.277879:0:67083:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:2.0:1775764858.277890:0:67083:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:2.0:1775764858.277907:0:67083:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764858.277931:0:67083:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764858.277936:0:67083:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764858.277941:0:67083:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1775764858.277950:0:67083:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:2.0:1775764858.277957:0:67083:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764858.277961:0:67083:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764858.277963:0:67083:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c24d 00000080:00010000:2.0:1775764858.277965:0:67083:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c24d from it ffffa60b0200fb38 00010000:00010000:2.0:1775764858.277968:0:67083:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1775764858.277974:0:67083:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1775764858.277982:0:67083:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:2.0:1775764858.277987:0:67083:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=. :VOLATILE:0000:4C6BE75:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b2383bb0f00, open_flags 280c2, mode 8180 opened 0 00000080:00200000:2.0:1775764858.277995:0:67083:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=. :VOLATILE:0000:4C6BE75:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000002:00010000:2.0:1775764858.278020:0:67083:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: . :VOLATILE:0000:4C6BE75:fd=09,[0x200003ab4:0x1e:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0500303 00010000:00010000:2.0:1775764858.278034:0:67083:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1775764858.278094:0:67083:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1775764858.278101:0:67083:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764858.278104:0:67083:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496000/0x73cfd86cfc17c24d lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052e32 expref: -99 pid: 67048 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1775764858.278147:0:67083:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67083 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.278154:0:67083:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67083 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764858.278167:0:67083:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764858.278177:0:67083:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7b80 x1862022098704640/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1088/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764858.278188:0:67083:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67083 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764858.278212:0:67083:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098704640:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764858.278398:0:67083:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bf00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764858.285326:0:67083:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098704640:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764858.285375:0:67083:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e94 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.285420:0:67083:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e94 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.285429:0:67083:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e94 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.285442:0:67083:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052e94 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.285504:0:67083:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e94 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.285522:0:67083:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052e94 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.285531:0:67083:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052e94 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.285540:0:67083:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052e94 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764858.285554:0:67083:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8b23b9ea7b80 x1862022098704640/t34359739379(34359739379) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1088/664 e 0 to 0 dl 1775764874 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764858.285569:0:67083:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7b80 x1862022098704640/t34359739379(34359739379) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1088/664 e 0 to 0 dl 1775764874 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764858.285586:0:67083:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e94 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764858.285603:0:67083:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e94 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.285626:0:67083:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/30 (0 0) 00000080:00200000:2.0:1775764858.285634:0:67083:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764858.285652:0:67083:0:(namei.c:1620:ll_create_it()) VFS Op:name=. :VOLATILE:0000:4C6BE75:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000080:00200000:2.0:1775764858.285695:0:67083:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab4:0x1e:0x0](ffff8b2390ec7448) 00000080:00200000:2.0:1775764858.285864:0:67083:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1e:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1775764858.285875:0:67083:0:(namei.c:1586:ll_create_node()) setting l_ast_data to inode [0x200003ab4:0x1e:0x0](ffff8b2390ec7448) 00000080:00010000:2.0:1775764858.285879:0:67083:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1e:0x0](ffff8b2390ec7448) for lock 0x73cfd86cfc17c269 00000080:00010000:2.0:1775764858.285891:0:67083:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1e:0x0](ffff8b2390ec7448) 00000080:00200008:2.0:1775764858.285911:0:67083:0:(file.c:1057:ll_file_open()) START file . :VOLATILE:0000:4C6BE75:fd=09:[0x200003ab4:0x1e:0x0](ffff8b2390ec7448), flags 500302 00000002:00100000:2.0:1775764858.285934:0:67083:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7b80 x1862022098704640/t34359739379(34359739379) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 608/664 e 0 to 0 dl 1775764874 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764858.285982:0:67083:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c269 from it ffff8b238504c8a0 00010000:00010000:2.0:1775764858.285986:0:67083:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052e94 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.285996:0:67083:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052e94 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764858.286044:0:67083:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x1e:0x0](ffff8b2390ec7448), inode mode 8180 mask 40 00000080:00200000:2.0:1775764858.286103:0:67083:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764858.286121:0:67083:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764858.286177:0:67083:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67083 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.286185:0:67083:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67083 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.286196:0:67083:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67083 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764858.286222:0:67083:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098704768:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764858.286572:0:67083:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764858.299332:0:67083:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098704768:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764858.299443:0:67083:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ea9 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.299596:0:67083:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ea9 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.299676:0:67083:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052ea9 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.299691:0:67083:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ea9 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.299788:0:67083:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052ea9 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.299798:0:67083:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052ea9 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.299811:0:67083:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052ea9 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764858.299824:0:67083:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea5880 x1862022098704768/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764874 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764858.299839:0:67083:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ea9 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.299853:0:67083:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764858.299860:0:67083:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764858.299901:0:67083:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764858.299908:0:67083:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c270 00000080:00010000:2.0:1775764858.299915:0:67083:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764858.299918:0:67083:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c270 from it ffffa60b0200fc10 00010000:00010000:2.0:1775764858.299921:0:67083:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052ea9 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.299931:0:67083:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ea9 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764858.299962:0:67083:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764858.299985:0:67083:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=. :VOLATILE:0000:4C6BE75:fd=09, flags=0 00000080:00200000:2.0:1775764858.299992:0:67083:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764858.300009:0:67083:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764858.300019:0:67083:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ea9 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764858.300034:0:67083:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ea9 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764858.300040:0:67083:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764858.300045:0:67083:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764858.300047:0:67083:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c270 00000080:00010000:2.0:1775764858.300051:0:67083:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c270 from it ffffa60b0200fd70 00010000:00010000:2.0:1775764858.300054:0:67083:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052ea9 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764858.300061:0:67083:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ea9 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764858.300068:0:67083:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:2.0:1775764858.300333:0:67083:0:(namei.c:2405:ll_unlink()) VFS Op:name=. :VOLATILE:0000:4C6BE75:fd=09, dir=[0x200000007:0x1:0x0](ffff8b238990cb08) 00010000:00010000:2.0:1775764858.301050:0:67083:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052ea9 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764858.301093:0:67083:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052e94 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764858.301114:0:67083:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1e:0x0]: losing layout lock 00010000:00010000:2.0:1775764858.301173:0:67083:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052e94 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764858.301181:0:67083:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052ea9 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764858.301193:0:67083:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1775764858.301196:0:67083:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c269 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052e94 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764858.301220:0:67083:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c270 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052ea9 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000020:00100000:2.0:1775764858.301242:0:67083:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764858.301333:0:67083:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5880 x1862022098705024/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/1032 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764858.301674:0:67083:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098705024:192.168.202.128@tcp:36:lhsmtool_posix.0 00000100:00100000:2.0:1775764858.301885:0:67083:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764858.313045:0:67083:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098705024:192.168.202.128@tcp:36:lhsmtool_posix.0 00000002:00010000:2.0:1775764858.313172:0:67083:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x0:0x0:0x0]) in obj [0x200003ab4:0x1e:0x0], intent: getxattr flags 0 00010000:00010000:2.0:1775764858.313254:0:67083:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c277 lrc: 3/1,0 mode: --/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67083 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.313265:0:67083:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c277 lrc: 3/1,0 mode: --/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67083 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.313276:0:67083:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c277 lrc: 3/1,0 mode: --/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67083 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764858.313299:0:67083:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098705152:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764858.313401:0:67083:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764858.318510:0:67083:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098705152:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764858.318546:0:67083:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c277 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ec5 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.318566:0:67083:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c277 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052ec5 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.318580:0:67083:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c277 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ec5 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764858.318598:0:67083:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=80 disposition=0, status=0 req@ffff8b23b9ea5880 x1862022098705152/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 568/1072 e 0 to 0 dl 1775764874 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764858.318611:0:67083:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c277 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ec5 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.318628:0:67083:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/30 (0 0) 00000080:00200000:2.0:1775764858.318635:0:67083:0:(namei.c:953:ll_intent_lock()) intent lock 128 on i1 [0x200003ab4:0x1e:0x0] suppgids 0 -1: rc 0 00000080:00010000:2.0:1775764858.318673:0:67083:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1e:0x0](ffff8b2390ec7448) for lock 0x73cfd86cfc17c277 00000080:00010000:2.0:1775764858.318685:0:67083:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c277 from it ffffa60b0200fc28 00010000:00010000:2.0:1775764858.318690:0:67083:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c277 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052ec5 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.318702:0:67083:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c277 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ec5 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1775764858.319180:0:67083:0:(llite_lib.c:2274:ll_setattr_raw()) START file . :VOLATILE:0000:4C6BE75:fd=09:[0x200003ab4:0x1e:0x0](ffff8b2390ec7448) current size 0, valid attrs 1846, mode 8b23, uid 0, gid 0, new size 0, atime 1775764858.000000000, mtime 1775764858.000000000, ctime 1775764858.000000000 00000100:00100000:2.0:1775764858.326579:0:67083:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098705408:192.168.202.128@tcp:36:lhsmtool_posix.0 00010000:00010000:2.0:1775764858.326653:0:67083:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 31 res 8589949620/30 (0 0) 00000080:00010000:2.0:1775764858.326671:0:67083:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x1e:0x0](ffff8b2390ec7448) 00000002:00010000:2.0:1775764858.326700:0:67083:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1e:0x0]) in obj [0x200003ab4:0x1e:0x0], intent: layout flags 0 00010000:00010000:2.0:1775764858.326774:0:67083:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c27e lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67083 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.326786:0:67083:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c27e lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67083 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.326798:0:67083:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c27e lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67083 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764858.326824:0:67083:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098705536:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764858.326894:0:67083:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b480 going to sleep for 11 seconds 00000100:00100000:2.0:1775764858.333392:0:67083:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098705536:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764858.333444:0:67083:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c27e lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052ed3 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.333478:0:67083:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c27e lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ed3 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764858.333496:0:67083:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea5880 x1862022098705536/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/400 e 0 to 0 dl 1775764874 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764858.333509:0:67083:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c27e lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ed3 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764858.333536:0:67083:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c27e lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ed3 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.333563:0:67083:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/30 (0 0) 00000080:00200000:2.0:1775764858.333571:0:67083:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x1e:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764858.333598:0:67083:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1e:0x0](ffff8b2390ec7448) for lock 0x73cfd86cfc17c27e 00000080:00010000:2.0:1775764858.333607:0:67083:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c27e from it ffffa60b0200fa68 00010000:00010000:2.0:1775764858.333611:0:67083:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c27e lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052ed3 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.333623:0:67083:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c27e lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ed3 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764858.333649:0:67083:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c27e lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ed3 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764858.333660:0:67083:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c27e lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ed3 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764858.333670:0:67083:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1e:0x0](000000002fe37071) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c27e lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052ed3 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764858.335817:0:67083:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1e:0x0]: layout version change: 4294967294 -> 0 00010000:00010000:2.0:1775764858.335870:0:67083:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c27e lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052ed3 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764858.335887:0:67083:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c27e lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052ed3 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764858.335943:0:67083:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1e:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764858.335959:0:67083:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764858.342961:0:67083:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1e:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1775764858.342995:0:67083:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file . :VOLATILE:0000:4C6BE75:fd=09:[0x200003ab4:0x1e:0x0](ffff8b2390ec7448) current size 0, valid attrs 1846, mode 8b23, uid 0, gid 0, new size 0, atime 1775764858.000000000, mtime 1775764858.000000000, ctime 1775764858.000000000, rc 0 00000080:00200008:2.0:1775764858.343076:0:67083:0:(file.c:447:ll_file_release()) START file 0x200003ab4:0x1b:0x0:[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), flags 1504000 00000002:00100000:2.0:1775764858.343161:0:67083:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864f0e00 x1862022098704256/t34359739378(34359739378) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764874 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764858.343175:0:67083:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764858.343180:0:67083:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4a80 x1862022098705920/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764858.343207:0:67083:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098705920:192.168.202.128@tcp:35:lhsmtool_posix.0 00000100:00100000:2.0:1775764858.343401:0:67083:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b480 going to sleep for 11 seconds 00000080:00200000:2.0:1775764858.375869:0:67083:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1e:0x0] updating i_size 0 i_blocks 1 00000020:00010000:2.0:1775764858.375876:0:67083:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b23983726c8 (ffff8b2383fb53f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764858.375880:0:67083:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23983726c8 (ffff8b2383fb53f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764858.375886:0:67083:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23849864b0 (ffff8b2383fb52d8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764858.375891:0:67083:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23983726c8 (ffff8b2383fb53f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764858.375894:0:67083:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23849864b0 (ffff8b2383fb53f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764858.375942:0:67083:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1e:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:2.0:1775764858.376053:0:67083:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=401866d8 arg=7fdbee2c3910 00800000:00000080:2.0:1775764858.376065:0:67083:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b0200fcf0 uarg=0000000000000000 00000002:00000080:2.0:1775764858.376076:0:67083:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b0200fcf0 uarg=0000000000000000 00000020:00100000:2.0:1775764858.376114:0:67083:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 57, max 7 00000100:00100000:2.0:1775764858.376126:0:67083:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6d80 x1862022098706688/t0(0) o57->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764858.376156:0:67083:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098706688:192.168.202.128@tcp:57:lhsmtool_posix.0 00000100:00100000:2.0:1775764858.376258:0:67083:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764858.378780:0:67083:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098706688:192.168.202.128@tcp:57:lhsmtool_posix.0 00000080:00200008:2.0:1775764858.379057:0:67083:0:(file.c:2612:do_file_write_iter()) START file . :VOLATILE:0000:4C6BE75:fd=09:[0x200003ab4:0x1e:0x0], ppos: 0, count: 22 00000080:00200000:2.0:1775764858.379077:0:67083:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 22 bytes 00000080:00200000:2.0:1775764858.379083:0:67083:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 22 00000080:00200000:2.0:1775764858.379126:0:67083:0:(file.c:2058:ll_file_io_generic()) . :VOLATILE:0000:4C6BE75:fd=09: write ppos: 0, bytes: 22 00000080:00200000:2.0:1775764858.379132:0:67083:0:(file.c:1815:ll_io_set_mirror()) . :VOLATILE:0000:4C6BE75:fd=09: desiginated mirror: 0 00000020:00200000:2.0:1775764858.379143:0:67083:0:(cl_io.c:214:cl_io_rw_init()) header@000000000b249891[0x110, 1, [0x200003ab4:0x1e:0x0] hash] 00000020:00200000:2.0:1775764858.379145:0:67083:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 22) 0 0 00000080:00200000:2.0:1775764858.379151:0:67083:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1e:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764858.379170:0:67083:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:2.0:1775764858.379179:0:67083:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 22) 22, 22 00020000:00200000:2.0:1775764858.379184:0:67083:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764858.379217:0:67083:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1775764858.379235:0:67083:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764858.379241:0:67083:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8bb0, osc lock ffff8b238a3f8be0, flags 0x0 00010000:00010000:2.0:1775764858.379262:0:67083:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2386938000 type 11 mode 2 res 10737419264/286 (0 4095) 00010000:00010000:2.0:1775764858.379317:0:67083:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.379329:0:67083:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.379340:0:67083:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764858.379382:0:67083:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098706816:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764858.379520:0:67083:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764858.382616:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764858.384203:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764858.384220:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000b9df339d/0x73cfd86cfc17c28c lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e03052ee8 expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764858.384237:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2381496000 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:2.0:1775764858.384247:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000b9df339d/0x73cfd86cfc17c28c lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x57b8ff0e03052ee8 expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764858.384330:0:46150:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764858.384348:0:46150:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775764858.384531:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000b9df339d/0x73cfd86cfc17c28c lrc: 2/0,0 mode: --/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e03052ee8 expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764858.384540:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764858.384587:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000b9df339d/0x73cfd86cfc17c28c lrc: 1/0,0 mode: --/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e03052ee8 expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764858.384597:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000b9df339d/0x73cfd86cfc17c28c lrc: 0/0,0 mode: --/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e03052ee8 expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764858.384941:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764858.384943:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764858.384946:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764858.388205:0:67083:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098706816:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764858.388253:0:67083:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.388270:0:67083:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.388279:0:67083:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764858.395756:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764858.396509:0:67083:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.396521:0:67083:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.396541:0:67083:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1775764858.396558:0:67083:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000020000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.396574:0:67083:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.396582:0:67083:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764858.396646:0:67083:0:(vvp_io.c:1286:vvp_io_write_start()) . :VOLATILE:0000:4C6BE75:fd=09: write [0, 22) append 0 append lockpos 0 00000080:00200000:2.0:1775764858.396671:0:67083:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 22 bytes 00000080:00200000:2.0:1775764858.396746:0:67083:0:(rw26.c:858:ll_write_end()) pos 0, len 22, copied 22 00000080:00200000:2.0:1775764858.396756:0:67083:0:(rw26.c:916:ll_write_end()) page@00000000dcc79985[3 0000000074d33138 2 1 0000000077ca0792] 00000080:00200000:2.0:1775764858.396760:0:67083:0:(rw26.c:916:ll_write_end()) vmpage @0000000055cd1448 17ffffc0001001 3:0 ffff8b23baeae1c0 0 lru 00000080:00200000:2.0:1775764858.396769:0:67083:0:(rw26.c:916:ll_write_end()) osc-page@00000000ab56b695 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 000000008819327c 00000000ed923912 > 3< 0 0 > 4< 0 0 8 17039360 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764858.396771:0:67083:0:(rw26.c:916:ll_write_end()) end page@00000000dcc79985 00000080:00200000:2.0:1775764858.396772:0:67083:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764858.396784:0:67083:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 22 prio 0 00000008:00010000:2.0:1775764858.396812:0:67083:0:(osc_cache.c:848:osc_extent_find()) ### extent: 00000000a8e7d926 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764858.396835:0:67083:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2390ec75f8, count 1, dirtied 1 00000080:00200000:2.0:1775764858.396840:0:67083:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 22 bytes, tot: 22 00000080:00200000:2.0:1775764858.396843:0:67083:0:(vvp_io.c:1353:vvp_io_write_start()) . :VOLATILE:0000:4C6BE75:fd=09: write bytes 0, result: 22 00000020:00010000:2.0:1775764858.396864:0:67083:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b8a8 (ffff8b23812ec3f8/0) at cl_lock_release():245 00000020:00010000:2.0:1775764858.396867:0:67083:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b8a8 (ffff8b23812ec3f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764858.396871:0:67083:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8bb0 (ffff8b23812ecea8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764858.396876:0:67083:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000020000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.396886:0:67083:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1775764858.396904:0:67083:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b8a8 (ffff8b23812ec3f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764858.396907:0:67083:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8bb0 (ffff8b23812ec3f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764858.396917:0:67083:0:(vvp_io.c:517:vvp_io_advance()) advancing 22 bytes 00000080:00200000:2.0:1775764858.396926:0:67083:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:2.0:1775764858.396966:0:67083:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1e:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764858.396971:0:67083:0:(file.c:2206:ll_file_io_generic()) . :VOLATILE:0000:4C6BE75:fd=09: 2 io complete with rc: 0, result: 22, restart: 0 00000080:00200000:2.0:1775764858.396980:0:67083:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1e:0x0] updating i_size 22 i_blocks 1 00000080:00200000:2.0:1775764858.396986:0:67083:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 22 00000080:00200000:2.0:1775764858.397336:0:67083:0:(file.c:5304:ll_fsync()) VFS Op:inode=[0x200003ab4:0x1e:0x0](ffff8b2390ec7448), start 0, end 9223372036854775807, datasync 0 00000080:00200000:2.0:1775764858.397389:0:67083:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764858.397401:0:67083:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:2.0:1775764858.397434:0:67083:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 00000000a8e7d926 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1775764858.397448:0:67083:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000a8e7d926 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764858.397633:0:67083:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00100000:2.0:1775764858.397728:0:67083:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea6d80 - object 0x280000400:286 offset 0<>22 00000100:00100000:2.0:1775764858.397830:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098707072:192.168.202.128@tcp:4:lhsmtool_posix.0 00000008:00010000:2.0:1775764858.415760:0:2378:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 00000000a8e7d926 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:2.0:1775764858.415835:0:2378:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a8e7d926 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1775764858.415866:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098707072:192.168.202.128@tcp:4:lhsmtool_posix.0 00000100:00100000:1.0:1775764858.488964:0:2376:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764858.488989:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098707328:192.168.202.128@tcp:16:lhsmtool_posix.0 00000080:00200000:2.0:1775764858.574676:0:67083:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1e:0x0] updating i_size 22 i_blocks 11 00000080:00200000:2.0:1775764858.574729:0:67083:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1e:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:2.0:1775764858.574770:0:67083:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x1e:0x0](ffff8b2390ec7448) cmd=c03066b0 arg=7fdbee2c2ac0 00000080:00200008:2.0:1775764858.574786:0:67083:0:(llite_lib.c:2274:ll_setattr_raw()) START file . :VOLATILE:0000:4C6BE75:fd=09:[0x200003ab4:0x1e:0x0](ffff8b2390ec7448) current size 22, valid attrs 1f0, mode 0, uid 0, gid 0, new size 0, atime 1775764853.000000000, mtime 1775764853.000000000, ctime 1775764853.000000000 00010000:00010000:2.0:1775764858.574841:0:67083:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c285 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052ee1 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764858.574880:0:67083:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c27e lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052ed3 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764858.574951:0:67083:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c285 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052ee1 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764858.574958:0:67083:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764858.574961:0:67083:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c285 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052ee1 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000020:00100000:2.0:1775764858.574988:0:67083:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764858.574998:0:67083:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4700 x1862022098707584/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764858.575030:0:67083:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098707584:192.168.202.128@tcp:36:lhsmtool_posix.0 00000100:00100000:2.0:1775764858.575147:0:67083:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764858.580865:0:67083:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098707584:192.168.202.128@tcp:36:lhsmtool_posix.0 00000080:00200000:2.0:1775764858.581026:0:67083:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1e:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764858.581046:0:67083:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:2.0:1775764858.581191:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764858.581211:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098707712:192.168.202.128@tcp:2:lhsmtool_posix.0 00000100:00100000:2.0:1775764858.584547:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098707712:192.168.202.128@tcp:2:lhsmtool_posix.0 00000100:00100000:1.0:1775764858.606090:0:46140:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764858.606111:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119083264 00000100:00100000:1.0:1775764858.606121:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119083264 00000100:00100000:1.0:1775764858.606208:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 300 00000100:00100000:1.0:1775764858.606219:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23b9ff3c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42325:x1862022119083264:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764858.606236:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c277 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03052ec5 expref: -99 pid: 67083 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764858.606329:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764858.606336:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23b9ff3c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42325:x1862022119083264:12345-192.168.202.128@tcp:104: Request processed in 119us (885us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764858.606344:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 300 00000100:00100000:2.0:1775764858.614227:0:67083:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67083:1862022098707968:192.168.202.128@tcp:55:lhsmtool_posix.0 00000080:00200008:2.0:1775764858.614297:0:67083:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file . :VOLATILE:0000:4C6BE75:fd=09:[0x200003ab4:0x1e:0x0](ffff8b2390ec7448) current size 22, valid attrs 1f0, mode 0, uid 0, gid 0, new size 0, atime 1775764853.000000000, mtime 1775764853.000000000, ctime 1775764853.000000000, rc 0 00000080:00200080:2.0:1775764858.614326:0:67083:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=400866d7 arg=7fdbe8000b78 00000080:00200000:2.0:1775764858.614444:0:67083:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1e:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764858.614496:0:67083:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:2.0:1775764858.614612:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764858.614633:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098708224:192.168.202.128@tcp:1:lhsmtool_posix.0 00000100:00100000:1.0:1775764858.619806:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119083392 00000100:00100000:1.0:1775764858.619818:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119083392 00000100:00100000:1.0:1775764858.619873:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 301 00000100:00100000:1.0:1775764858.619883:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dafb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42946:x1862022119083392:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764858.619900:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800420000020000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764858.619991:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764858.619997:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dafb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42946:x1862022119083392:12345-192.168.202.128@tcp:104: Request processed in 117us (615us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764858.620511:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 301 00010000:00010000:2.0:1775764858.621676:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764858.621692:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800420000020000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764858.621711:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2380030a00 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:2.0:1775764858.621722:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800428400020000 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764858.621818:0:46150:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764858.621913:0:46150:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775764858.622020:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764858.622030:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764858.622084:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764858.622099:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000a7969a8b/0x73cfd86cfc17c293 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e03052eef expref: -99 pid: 67083 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764858.622131:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764858.622133:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764858.622147:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764858.638685:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119083520 00000100:00100000:1.0:1775764858.638696:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119083520 00000100:00100000:1.0:1775764858.638873:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 302 00000100:00100000:1.0:1775764858.638884:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dadf80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119083520:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764858.638902:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c27e lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1e:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03052ed3 expref: -99 pid: 67083 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764858.638987:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764858.638993:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dadf80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119083520:12345-192.168.202.128@tcp:104: Request processed in 112us (520us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764858.639009:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 302 00000100:00100000:2.0:1775764858.647034:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098708736:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764858.647536:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119083904 00000100:00100000:1.0:1775764858.647545:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119083904 00000100:00100000:1.0:1775764858.647620:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 303 00000100:00100000:1.0:1775764858.647630:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dac380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119083904:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764858.647647:0:46139:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 5/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764858.647666:0:46139:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 5/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764858.647675:0:46139:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 5/1,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764858.647685:0:46139:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 5/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764858.647694:0:46139:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 6/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764858.647714:0:46139:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 6/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x80000000000 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764858.647749:0:46139:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b2380030000) 00000100:00100000:1.0:1775764858.647821:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dac380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119083904:12345-192.168.202.128@tcp:105: Request processed in 192us (1016us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764858.647828:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 303 00010000:00010000:2.0:1775764858.648611:0:67081:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.648625:0:67081:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764858.648649:0:67081:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea6a00 x1862022098703232/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/344 e 0 to 0 dl 1775764873 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764858.648661:0:67081:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.648674:0:67081:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/27 (0 0) 00000080:00200000:2.0:1775764858.648680:0:67081:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x1b:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764858.648713:0:67081:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c25b 00000080:00010000:2.0:1775764858.648722:0:67081:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c25b from it ffffa60b01f0bc18 00010000:00010000:2.0:1775764858.648726:0:67081:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764858.648740:0:67081:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764858.648765:0:67081:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764858.648775:0:67081:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764858.648782:0:67081:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1b:0x0](000000000e8c9725) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00100000:2.0:1775764858.648830:0:67081:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67081:1862022098708864:192.168.202.128@tcp:49:lfs.0 00000100:00100000:2.0:1775764858.648906:0:67081:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764858.662850:0:67081:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67081:1862022098708864:192.168.202.128@tcp:49:lfs.0 00010000:00010000:2.0:1775764858.662962:0:67081:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764858.662977:0:67081:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764858.663101:0:67081:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764858.663111:0:67081:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764858.663149:0:67081:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764858.663165:0:67081:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764858.663172:0:67081:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1b:0x0](000000000e8c9725) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764858.663215:0:67081:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764858.663219:0:67081:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764858.663290:0:67081:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1b:0x0]: layout version change: 4294967294 -> 2 00010000:00010000:2.0:1775764858.663299:0:67081:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764858.663307:0:67081:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00800000:00000080:2.0:1775764858.663817:0:67081:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d9 len=56 karg=ffff8b238872eb80 uarg=0000000000000000 00000002:00000080:2.0:1775764858.663828:0:67081:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d9 len=56 karg=ffff8b238872eb80 uarg=0000000000000000 00000020:00100000:2.0:1775764858.663855:0:67081:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 2 is allocated opc 58, max 7 00000100:00100000:2.0:1775764858.663863:0:67081:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6a00 x1862022098709376/t0(0) o58->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764858.663884:0:67081:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67081:1862022098709376:192.168.202.128@tcp:58:lfs.0 00000100:00100000:2.0:1775764858.663985:0:67081:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764858.673167:0:67081:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67081:1862022098709376:192.168.202.128@tcp:58:lfs.0 00000080:00200008:2.0:1775764858.673204:0:67081:0:(file.c:447:ll_file_release()) START file f27.sanity-pcc:[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), flags 154002 00010000:00010000:2.0:1775764858.807663:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1775764858.935778:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x16:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764858.935791:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764858.935851:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x16:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:1.0:1775764859.675253:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000080:00200080:2.0:1775764859.676576:0:67096:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=c0086696 arg=7fdbe8000d20 00800000:00000080:2.0:1775764859.676599:0:67096:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=c0086696 len=4128 karg=ffff8b2391290000 uarg=0000000000000000 00000002:00000080:2.0:1775764859.676607:0:67096:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=c0086696 len=4128 karg=ffff8b2391290000 uarg=0000000000000000 00000002:00000080:2.0:1775764859.676614:0:67096:0:(mdc_request.c:1738:mdc_ioc_fid2path()) path get [0x200003ab4:0x1b:0x0] from 18446744073709551615 #0 00000100:00100000:2.0:1775764859.676660:0:67096:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67096:1862022098709760:192.168.202.128@tcp:53:lhsmtool_posix.0 00000100:00100000:2.0:1775764859.676774:0:67096:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764859.690224:0:67096:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67096:1862022098709760:192.168.202.128@tcp:53:lhsmtool_posix.0 00000002:00000080:2.0:1775764859.690265:0:67096:0:(mdc_request.c:1758:mdc_ioc_fid2path()) path got [0x200000007:0x1:0x0] from 18446744073709551615 #0: f27.sanity-pcc 00000080:00200000:2.0:1775764859.690421:0:67096:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000002:0x2:0x0](ffff8b2390ec5348), inode mode 4040 mask 1 00000080:00200000:2.0:1775764859.690443:0:67096:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=0x200003ab4:0x1b:0x0, dir=[0x200000002:0x2:0x0](ffff8b2390ec5348), file ffff8b2392ce7800, open_flags 68800, mode 0 opened 0 00000080:00200000:2.0:1775764859.690490:0:67096:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=0x200003ab4:0x1b:0x0, dir=[0x200000002:0x2:0x0](ffff8b2390ec5348), intent=open 00000002:00010000:2.0:1775764859.690525:0:67096:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1b:0x0]) in obj [0x200000002:0x2:0x0], intent: open flags 041504001 00010000:00010000:2.0:1775764859.690596:0:67096:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c2a8 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67096 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764859.690605:0:67096:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c2a8 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67096 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764859.690617:0:67096:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764859.690625:0:67096:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6a00 x1862022098710016/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764859.690635:0:67096:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c2a8 lrc: 3/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67096 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764859.690681:0:67096:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67096:1862022098710016:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764859.690879:0:67096:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b780 going to sleep for 11 seconds 00000100:00100000:2.0:1775764859.697403:0:67096:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67096:1862022098710016:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764859.697431:0:67096:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c2a8 lrc: 4/1,0 mode: --/CR res: [0x200000002:0x2:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f58 expref: -99 pid: 67096 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764859.697450:0:67096:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c2a8 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f58 expref: -99 pid: 67096 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764859.697494:0:67096:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c2a8 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f58 expref: -99 pid: 67096 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764859.697506:0:67096:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c2a8 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052f58 expref: -99 pid: 67096 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764859.697519:0:67096:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c2a8 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f58 expref: -99 pid: 67096 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764859.697532:0:67096:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c2a8 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052f58 expref: -99 pid: 67096 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764859.697541:0:67096:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c2a8 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052f58 expref: -99 pid: 67096 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764859.697548:0:67096:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c2a8 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052f58 expref: -99 pid: 67096 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764859.697565:0:67096:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b23b9ea6a00 x1862022098710016/t34359739391(34359739391) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764875 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764859.697577:0:67096:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea6a00 x1862022098710016/t34359739391(34359739391) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764875 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764859.697591:0:67096:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c2a8 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f58 expref: -99 pid: 67096 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764859.697605:0:67096:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c2a8 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f58 expref: -99 pid: 67096 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764859.697618:0:67096:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c2a1 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052f4a expref: -99 pid: 67085 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764859.697630:0:67096:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c2a1 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052f4a expref: -99 pid: 67085 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764859.697637:0:67096:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c2a8 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f58 expref: -99 pid: 67096 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764859.697643:0:67096:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c2a8 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e03052f58 expref: -99 pid: 67096 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764859.697649:0:67096:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c2a8 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e03052f58 expref: -99 pid: 67096 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764859.697677:0:67096:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:2.0:1775764859.697687:0:67096:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000002:0x2:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764859.697797:0:67096:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88): rc = 0 00000080:00200000:2.0:1775764859.697820:0:67096:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1b:0x0]: layout version change: 2 -> 2 00000080:00010000:2.0:1775764859.697831:0:67096:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c2a1 00000080:00010000:2.0:1775764859.697844:0:67096:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) 00000080:00200008:2.0:1775764859.697859:0:67096:0:(file.c:1057:ll_file_open()) START file 0x200003ab4:0x1b:0x0:[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), flags 1504000 00000002:00100000:2.0:1775764859.697880:0:67096:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea6a00 x1862022098710016/t34359739391(34359739391) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764875 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764859.697931:0:67096:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c2a1 from it ffff8b238504c1e0 00010000:00010000:2.0:1775764859.697936:0:67096:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c2a1 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052f4a expref: -99 pid: 67085 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764859.697945:0:67096:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c2a1 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052f4a expref: -99 pid: 67085 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764859.697970:0:67096:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), inode mode 81a4 mask 44 00000080:00200080:2.0:1775764859.698256:0:67096:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=400866d7 arg=7fdbe8000b78 00800000:00000080:2.0:1775764859.698268:0:67096:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b0202fcf0 uarg=0000000000000000 00000002:00000080:2.0:1775764859.698275:0:67096:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b0202fcf0 uarg=0000000000000000 00000020:00100000:2.0:1775764859.698303:0:67096:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 57, max 7 00000100:00100000:2.0:1775764859.698311:0:67096:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5c00 x1862022098710272/t0(0) o57->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764859.698338:0:67096:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67096:1862022098710272:192.168.202.128@tcp:57:lhsmtool_posix.0 00000100:00100000:2.0:1775764859.698442:0:67096:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bd80 going to sleep for 11 seconds 00010000:00010000:2.0:1775764859.831654:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764860.866488:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764861.884086:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1775764862.551360:0:67021:0:(file.c:447:ll_file_release()) START file f27.sanity-pcc:[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), flags 100002 00000002:00100000:1.0:1775764862.551442:0:67021:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fd180 x1862022098702080/t34359739375(34359739375) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764869 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764862.551864:0:67021:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764862.551872:0:67021:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ff800 x1862022098710912/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764862.551900:0:67021:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:67021:1862022098710912:192.168.202.128@tcp:35:multiop.0 00000100:00100000:1.0:1775764862.553310:0:67021:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654f00 going to sleep for 11 seconds 00000080:00200000:1.0:1775764862.560049:0:67142:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764862.560072:0:67142:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764862.560087:0:67142:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17c29a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f2e expref: -99 pid: 67085 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764862.560098:0:67142:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17c29a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f2e expref: -99 pid: 67085 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764862.560104:0:67142:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764862.560111:0:67142:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764862.560113:0:67142:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c29a 00000080:00010000:1.0:1775764862.560119:0:67142:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c29a from it ffffa60b02197b58 00010000:00010000:1.0:1775764862.560122:0:67142:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17c29a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052f2e expref: -99 pid: 67085 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764862.560128:0:67142:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17c29a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f2e expref: -99 pid: 67085 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764862.560139:0:67142:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764862.560144:0:67142:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f27.sanity-pcc, flags=8 00000080:00200008:1.0:1775764862.560149:0:67142:0:(file.c:6226:ll_getattr_dentry()) START file f27.sanity-pcc:[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764862.560152:0:67142:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88),name=f27.sanity-pcc 00000002:00010000:1.0:1775764862.560159:0:67142:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1b:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764862.560164:0:67142:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c2a1 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052f4a expref: -99 pid: 67085 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764862.560170:0:67142:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c2a1 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052f4a expref: -99 pid: 67085 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764862.560173:0:67142:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764862.560175:0:67142:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) 00000080:00010000:1.0:1775764862.560177:0:67142:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c2a1 00000080:00010000:1.0:1775764862.560178:0:67142:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c2a1 from it ffffa60b02197cb8 00010000:00010000:1.0:1775764862.560180:0:67142:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c2a1 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052f4a expref: -99 pid: 67085 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764862.560184:0:67142:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c2a1 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052f4a expref: -99 pid: 67085 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764862.560219:0:67142:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1b:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764862.560227:0:67142:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x1b:0x0] 00000008:00010000:1.0:1775764862.560248:0:67142:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764862.560251:0:67142:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5430, osc lock ffff8b2382cf7a20, flags 0x400001002 00010000:00010000:1.0:1775764862.560294:0:67142:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000006bafc880/0x73cfd86cfc17c2af lrc: 3/1,0 mode: --/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 67142 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764862.560304:0:67142:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000006bafc880/0x73cfd86cfc17c2af lrc: 3/1,0 mode: --/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67142 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1775764862.560357:0:67142:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:1.0:1775764862.560372:0:2377:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1775764862.560395:0:2377:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098711040:192.168.202.128@tcp:101:rm.0 00010000:00010000:1.0:1775764862.563692:0:2377:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000006bafc880/0x73cfd86cfc17c2af lrc: 6/2,0 mode: --/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 67142 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764862.563713:0:2377:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000006bafc880/0x73cfd86cfc17c2af lrc: 6/2,0 mode: --/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f6d expref: -99 pid: 67142 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764862.563724:0:2377:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000006bafc880/0x73cfd86cfc17c2af lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052f6d expref: -99 pid: 67142 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764862.563732:0:2377:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000006bafc880/0x73cfd86cfc17c2af lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f6d expref: -99 pid: 67142 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1775764862.563743:0:2377:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 22, setting rss=22; kms=22, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000006bafc880/0x73cfd86cfc17c2af lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052f6d expref: -99 pid: 67142 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1775764862.563750:0:2377:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1775764862.563763:0:2377:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000006bafc880/0x73cfd86cfc17c2af lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052f6d expref: -99 pid: 67142 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764862.563771:0:2377:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000006bafc880/0x73cfd86cfc17c2af lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052f6d expref: -99 pid: 67142 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764862.563779:0:2377:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000006bafc880/0x73cfd86cfc17c2af lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052f6d expref: -99 pid: 67142 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764862.563792:0:2377:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000006bafc880/0x73cfd86cfc17c2af lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052f6d expref: -99 pid: 67142 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1775764862.563812:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098711040:192.168.202.128@tcp:101:rm.0 00000080:00200000:1.0:1775764862.566433:0:67142:0:(namei.c:2405:ll_unlink()) VFS Op:name=f27.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08) 00010000:00010000:1.0:1775764862.566529:0:67142:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17c29a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052f2e expref: -99 pid: 67085 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764862.566560:0:67142:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c2a1 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052f4a expref: -99 pid: 67085 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764862.566578:0:67142:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1b:0x0]: losing layout lock 00010000:00010000:1.0:1775764862.566603:0:67142:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764862.566620:0:67142:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1b:0x0]: losing layout lock 00010000:00010000:1.0:1775764862.566660:0:67142:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c2a1 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052f4a expref: -99 pid: 67085 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764862.566679:0:67142:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764862.566685:0:67142:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17c29a lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052f2e expref: -99 pid: 67085 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764862.566691:0:67142:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:1.0:1775764862.566694:0:67142:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c2a1 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052f4a expref: -99 pid: 67085 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764862.566705:0:67142:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c25b lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x1b:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052e5c expref: -99 pid: 67081 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764862.566719:0:67142:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17c29a lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052f2e expref: -99 pid: 67085 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000020:00100000:1.0:1775764862.566738:0:67142:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1775764862.566747:0:67142:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fea00 x1862022098711296/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 504/1032 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764862.566767:0:67142:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job rm:fcf15866-4739-4595-8fe1-44abde7ee9d7:67142:1862022098711296:192.168.202.128@tcp:36:rm.0 00000100:00100000:1.0:1775764862.566830:0:67142:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654840 going to sleep for 11 seconds 00000100:00100000:1.0:1775764862.570524:0:67142:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job rm:fcf15866-4739-4595-8fe1-44abde7ee9d7:67142:1862022098711296:192.168.202.128@tcp:36:rm.0 00000080:00200000:1.0:1775764862.570613:0:67142:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764862.570625:0:67142:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764862.570679:0:67142:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764862.570695:0:67142:0:(llite_lib.c:1913:ll_clear_inode()) VFS Op:inode=[0x200003ab4:0x1b:0x0](ffff8b2390ec5b88) 00000008:00010000:1.0:1775764862.570731:0:67142:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 22 blocks 11 [cma]time: 1775764853 1775764853 1775764853 by oinfo size 22 blocks 11 [cma]time 1775764853 1775764853 1775764853 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000006bafc880/0x73cfd86cfc17c2af lrc: 1/0,0 mode: PR/PR res: [0x280000400:0x11e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052f6d expref: -99 pid: 67142 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764862.571993:0:66748:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764862.572016:0:66748:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764862.572064:0:66748:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66748 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.572070:0:66748:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66748 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.572078:0:66748:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66748 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764862.572097:0:66748:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:66748:1862022098711424:192.168.202.128@tcp:101:bash.0 00000100:00100000:2.0:1775764862.572170:0:66748:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b840 going to sleep for 11 seconds 00000100:00100000:2.0:1775764862.574285:0:66748:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:66748:1862022098711424:192.168.202.128@tcp:101:bash.0 00010000:00010000:2.0:1775764862.574309:0:66748:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f97 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.574319:0:66748:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f97 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.574342:0:66748:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052f97 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.574353:0:66748:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f97 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.574364:0:66748:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052f97 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.574389:0:66748:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052f97 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.574396:0:66748:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052f97 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764862.574408:0:66748:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea4a80 x1862022098711424/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764878 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764862.574419:0:66748:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f97 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.574428:0:66748:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764862.574432:0:66748:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764862.574487:0:66748:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764862.574495:0:66748:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c2b6 00000080:00010000:2.0:1775764862.574501:0:66748:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764862.574504:0:66748:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c2b6 from it ffffa60b01e0bbf8 00010000:00010000:2.0:1775764862.574507:0:66748:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052f97 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.574516:0:66748:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f97 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764862.574538:0:66748:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764862.574557:0:66748:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764862.574566:0:66748:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764862.574573:0:66748:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f97 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764862.574580:0:66748:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f97 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764862.574584:0:66748:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764862.574587:0:66748:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764862.574588:0:66748:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c2b6 00000080:00010000:2.0:1775764862.574590:0:66748:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c2b6 from it ffffa60b01e0bb38 00010000:00010000:2.0:1775764862.574592:0:66748:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052f97 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764862.574596:0:66748:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052f97 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764862.574600:0:66748:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:2.0:1775764862.574604:0:66748:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f27.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b2383103b00, open_flags 8241, mode 81b6 opened 0 00000080:00200000:2.0:1775764862.574611:0:66748:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f27.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000002:00010000:2.0:1775764862.574626:0:66748:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f27.sanity-pcc,[0x200003ab4:0x1f:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0101102 00010000:00010000:2.0:1775764862.574635:0:66748:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052f97 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764862.574670:0:66748:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052f97 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764862.574674:0:66748:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764862.574676:0:66748:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2b6 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052f97 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764862.574703:0:66748:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c2bd lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66748 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.574707:0:66748:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c2bd lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66748 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764862.574715:0:66748:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764862.574721:0:66748:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4a80 x1862022098711680/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764862.574729:0:66748:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c2bd lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66748 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764862.574748:0:66748:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:66748:1862022098711680:192.168.202.128@tcp:101:bash.0 00000100:00100000:2.0:1775764862.574867:0:66748:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b840 going to sleep for 11 seconds 00000100:00100000:2.0:1775764862.578193:0:66748:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:66748:1862022098711680:192.168.202.128@tcp:101:bash.0 00010000:00010000:2.0:1775764862.578220:0:66748:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c2bd lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052fb3 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.578249:0:66748:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c2bd lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052fb3 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.578258:0:66748:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c2bd lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052fb3 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.578270:0:66748:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c2bd lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052fb3 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.578287:0:66748:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c2bd lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052fb3 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.578300:0:66748:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c2bd lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052fb3 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.578308:0:66748:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c2bd lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052fb3 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.578315:0:66748:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c2bd lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052fb3 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764862.578381:0:66748:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8b23b9ea4a80 x1862022098711680/t34359739396(34359739396) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764878 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764862.578392:0:66748:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4a80 x1862022098711680/t34359739396(34359739396) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764878 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764862.578405:0:66748:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c2bd lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052fb3 expref: -99 pid: 66748 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764862.578417:0:66748:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c2bd lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052fb3 expref: -99 pid: 66748 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.578426:0:66748:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/31 (0 0) 00000080:00200000:2.0:1775764862.578431:0:66748:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764862.578444:0:66748:0:(namei.c:1620:ll_create_it()) VFS Op:name=f27.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000080:00200000:2.0:1775764862.578679:0:66748:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) 00000080:00200000:2.0:1775764862.578812:0:66748:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1f:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1775764862.578823:0:66748:0:(namei.c:1586:ll_create_node()) setting l_ast_data to inode [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) 00000080:00010000:2.0:1775764862.578827:0:66748:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c2bd 00000080:00010000:2.0:1775764862.578837:0:66748:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) 00000080:00200008:2.0:1775764862.578852:0:66748:0:(file.c:1057:ll_file_open()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), flags 101101 00000002:00100000:2.0:1775764862.578872:0:66748:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4a80 x1862022098711680/t34359739396(34359739396) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764878 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764862.578907:0:66748:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c2bd from it ffff8b238504c840 00010000:00010000:2.0:1775764862.578912:0:66748:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c2bd lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052fb3 expref: -99 pid: 66748 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.578921:0:66748:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031600/0x73cfd86cfc17c2bd lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03052fb3 expref: -99 pid: 66748 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764862.578961:0:66748:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), inode mode 81a4 mask 40 00000080:00200008:2.0:1775764862.579057:0:66748:0:(file.c:2612:do_file_write_iter()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0], ppos: 0, count: 22 00000080:00200000:2.0:1775764862.579071:0:66748:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 22 bytes 00000080:00200000:2.0:1775764862.579078:0:66748:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 22 00000080:00200000:2.0:1775764862.579089:0:66748:0:(file.c:2058:ll_file_io_generic()) f27.sanity-pcc: write ppos: 0, bytes: 22 00000080:00200000:2.0:1775764862.579094:0:66748:0:(file.c:1815:ll_io_set_mirror()) f27.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764862.579102:0:66748:0:(cl_io.c:214:cl_io_rw_init()) header@000000000ca6206a[0x110, 1, [0x200003ab4:0x1f:0x0] hash] 00000020:00200000:2.0:1775764862.579104:0:66748:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 22) 0 0 00000080:00200000:2.0:1775764862.579109:0:66748:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1f:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764862.579124:0:66748:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:2.0:1775764862.579130:0:66748:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 22) 22, 22 00020000:00200000:2.0:1775764862.579134:0:66748:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764862.579160:0:66748:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1775764862.579173:0:66748:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764862.579179:0:66748:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8fb0, osc lock ffff8b238a3f8be0, flags 0x0 00010000:00010000:2.0:1775764862.579225:0:66748:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.579236:0:66748:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.579246:0:66748:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764862.579266:0:66748:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:66748:1862022098711808:192.168.202.128@tcp:101:bash.0 00000100:00100000:2.0:1775764862.579785:0:66748:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764862.581830:0:66748:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:66748:1862022098711808:192.168.202.128@tcp:101:bash.0 00010000:00010000:2.0:1775764862.581851:0:66748:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.581865:0:66748:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.581873:0:66748:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.581883:0:66748:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1775764862.581891:0:66748:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.581901:0:66748:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.581908:0:66748:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764862.581930:0:66748:0:(vvp_io.c:1286:vvp_io_write_start()) f27.sanity-pcc: write [0, 22) append 0 append lockpos 0 00000080:00200000:2.0:1775764862.581946:0:66748:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 22 bytes 00000080:00200000:2.0:1775764862.582060:0:66748:0:(rw26.c:858:ll_write_end()) pos 0, len 22, copied 22 00000080:00200000:2.0:1775764862.582069:0:66748:0:(rw26.c:916:ll_write_end()) page@00000000dcc79985[3 0000000020e697ef 2 1 00000000dad4ab84] 00000080:00200000:2.0:1775764862.582071:0:66748:0:(rw26.c:916:ll_write_end()) vmpage @00000000f0b365f0 17ffffc0001001 3:0 ffff8b23baeae1c0 0 lru 00000080:00200000:2.0:1775764862.582077:0:66748:0:(rw26.c:916:ll_write_end()) osc-page@00000000ab56b695 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 000000003596ffcd 000000002f534689 > 3< 0 0 > 4< 0 0 8 20447232 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1775764862.582078:0:66748:0:(rw26.c:916:ll_write_end()) end page@00000000dcc79985 00000080:00200000:2.0:1775764862.582079:0:66748:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764862.582083:0:66748:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 22 prio 0 00000008:00010000:2.0:1775764862.582099:0:66748:0:(osc_cache.c:848:osc_extent_find()) ### extent: 00000000a8e7d926 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1775764862.582115:0:66748:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2390ec5d38, count 1, dirtied 1 00000080:00200000:2.0:1775764862.582118:0:66748:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 22 bytes, tot: 22 00000080:00200000:2.0:1775764862.582120:0:66748:0:(vvp_io.c:1353:vvp_io_write_start()) f27.sanity-pcc: write bytes 0, result: 22 00000020:00010000:2.0:1775764862.582135:0:66748:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b308 (ffff8b23812ec638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764862.582137:0:66748:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b308 (ffff8b23812ec638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764862.582139:0:66748:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8fb0 (ffff8b23812ec098/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764862.582143:0:66748:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.582149:0:66748:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1775764862.582164:0:66748:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b308 (ffff8b23812ec638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764862.582166:0:66748:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8fb0 (ffff8b23812ec638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764862.582171:0:66748:0:(vvp_io.c:517:vvp_io_advance()) advancing 22 bytes 00000080:00200000:2.0:1775764862.582177:0:66748:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:2.0:1775764862.582202:0:66748:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1f:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764862.582205:0:66748:0:(file.c:2206:ll_file_io_generic()) f27.sanity-pcc: 2 io complete with rc: 0, result: 22, restart: 0 00000080:00200000:2.0:1775764862.582212:0:66748:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1f:0x0] updating i_size 22 i_blocks 1 00000080:00200000:2.0:1775764862.582218:0:66748:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 22 00000080:00200008:2.0:1775764862.582271:0:66748:0:(file.c:447:ll_file_release()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), flags 100001 00000002:00100000:2.0:1775764862.582359:0:66748:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea4a80 x1862022098711680/t34359739396(34359739396) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764878 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764862.582370:0:66748:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764862.582375:0:66748:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7480 x1862022098712064/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764862.582400:0:66748:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:66748:1862022098712064:192.168.202.128@tcp:35:bash.0 00000100:00100000:2.0:1775764862.582541:0:66748:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764862.584161:0:66748:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:66748:1862022098712064:192.168.202.128@tcp:35:bash.0 00000002:00100000:2.0:1775764862.584182:0:66748:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea4a80 x1862022098711680/t34359739396(34359739396) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764878 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764862.619709:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:2.0:1775764862.620254:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098712576:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764862.622335:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098712576:192.168.202.128@tcp:103:ldlm_bl.0 00000080:00200008:2.0:1775764862.627144:0:67148:0:(file.c:2454:do_file_read_iter()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0], ppos: 0, count: 1048576 00000080:00200000:2.0:1775764862.627175:0:67148:0:(file.c:2058:ll_file_io_generic()) f27.sanity-pcc: read ppos: 0, bytes: 1048576 00000080:00200000:2.0:1775764862.627179:0:67148:0:(file.c:1815:ll_io_set_mirror()) f27.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764862.627184:0:67148:0:(cl_io.c:214:cl_io_rw_init()) header@000000000ca6206a[0x110, 2, [0x200003ab4:0x1f:0x0] hash] 00000020:00200000:2.0:1775764862.627186:0:67148:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 1048576) 2048 0 00000080:00200000:2.0:1775764862.627190:0:67148:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1f:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00010000:00010000:2.0:1775764862.627204:0:67148:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 31 res 8589949620/31 (0 0) 00000080:00010000:2.0:1775764862.627211:0:67148:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) 00000002:00010000:2.0:1775764862.627218:0:67148:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1f:0x0]) in obj [0x200003ab4:0x1f:0x0], intent: layout flags 0 00010000:00010000:2.0:1775764862.627253:0:67148:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2e0 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67148 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.627259:0:67148:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2e0 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67148 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.627266:0:67148:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2e0 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67148 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764862.627281:0:67148:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67148:1862022098712960:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764862.627378:0:67148:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764862.629171:0:67148:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67148:1862022098712960:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764862.629202:0:67148:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2e0 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053000 expref: -99 pid: 67148 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.629223:0:67148:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2e0 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053000 expref: -99 pid: 67148 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764862.629233:0:67148:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea4000 x1862022098712960/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/400 e 0 to 0 dl 1775764878 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764862.629241:0:67148:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2e0 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053000 expref: -99 pid: 67148 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764862.629252:0:67148:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2e0 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053000 expref: -99 pid: 67148 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.629262:0:67148:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/31 (0 0) 00000080:00200000:2.0:1775764862.629266:0:67148:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x1f:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764862.629279:0:67148:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c2e0 00000080:00010000:2.0:1775764862.629291:0:67148:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c2e0 from it ffffa60b02187778 00010000:00010000:2.0:1775764862.629295:0:67148:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2e0 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053000 expref: -99 pid: 67148 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.629304:0:67148:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2e0 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053000 expref: -99 pid: 67148 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764862.629332:0:67148:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2e0 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053000 expref: -99 pid: 67148 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764862.629343:0:67148:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2e0 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053000 expref: -99 pid: 67148 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764862.629352:0:67148:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1f:0x0](000000000e8c9725) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2e0 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053000 expref: -99 pid: 67148 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764862.629416:0:67148:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1f:0x0]: layout version change: 4294967294 -> 0 00010000:00010000:2.0:1775764862.629438:0:67148:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2e0 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03053000 expref: -99 pid: 67148 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764862.629446:0:67148:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2e0 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053000 expref: -99 pid: 67148 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00020000:00200000:2.0:1775764862.629625:0:67148:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:2.0:1775764862.629628:0:67148:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764862.629649:0:67148:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:2.0:1775764862.629659:0:67148:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 1 00000008:00010000:2.0:1775764862.629663:0:67148:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8fb0, osc lock ffff8b238a3f8be0, flags 0x40000 00010000:00010000:2.0:1775764862.629675:0:67148:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764862.629682:0:67148:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764862.629690:0:67148:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764862.629695:0:67148:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764862.629699:0:67148:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764862.629705:0:67148:0:(vvp_io.c:826:vvp_io_read_start()) f27.sanity-pcc: read [0, 1048576) 00000001:00010000:2.0:1775764862.629715:0:67148:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x1f:0x0] 00000008:00010000:2.0:1775764862.629720:0:67148:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764862.629722:0:67148:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8130, osc lock ffff8b238a3f8130, flags 0x400001002 00010000:00010000:2.0:1775764862.629728:0:67148:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764862.629740:0:67148:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1775764862.629748:0:67148:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764862.629752:0:67148:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 8/1,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764862.629759:0:67148:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 7/1,0 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1775764862.629765:0:67148:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764862.629772:0:67148:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1f:0x0] updating i_size 22 i_blocks 1 00000020:00010000:2.0:1775764862.629775:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c6d8 (ffff8b23812ec638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764862.629778:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c6d8 (ffff8b23812ec638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764862.629781:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8130 (ffff8b23812ec098/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764862.629785:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c6d8 (ffff8b23812ec638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764862.629787:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8130 (ffff8b23812ec638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764862.629795:0:67148:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:0 00000020:00010000:2.0:1775764862.629808:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b308 (ffff8b23812ec638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764862.629810:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b308 (ffff8b23812ec638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764862.629811:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8fb0 (ffff8b23812ec098/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764862.629814:0:67148:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764862.629819:0:67148:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00010000:2.0:1775764862.629826:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b308 (ffff8b23812ec638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764862.629828:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8fb0 (ffff8b23812ec638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764862.629831:0:67148:0:(vvp_io.c:517:vvp_io_advance()) advancing 22 bytes 00000080:00200000:2.0:1775764862.629843:0:67148:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1f:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764862.629846:0:67148:0:(file.c:2206:ll_file_io_generic()) f27.sanity-pcc: 1 io complete with rc: 0, result: 22, restart: 0 00000080:00200000:2.0:1775764862.629849:0:67148:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 22 00000080:00200008:2.0:1775764862.629939:0:67148:0:(file.c:2454:do_file_read_iter()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0], ppos: 22, count: 1048576 00000080:00200000:2.0:1775764862.629953:0:67148:0:(file.c:2058:ll_file_io_generic()) f27.sanity-pcc: read ppos: 22, bytes: 1048576 00000080:00200000:2.0:1775764862.629956:0:67148:0:(file.c:1815:ll_io_set_mirror()) f27.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764862.629960:0:67148:0:(cl_io.c:214:cl_io_rw_init()) header@000000000ca6206a[0x110, 2, [0x200003ab4:0x1f:0x0] hash] 00000020:00200000:2.0:1775764862.629962:0:67148:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [22, 1048598) 2048 0 00000080:00200000:2.0:1775764862.630053:0:67148:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1f:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764862.630061:0:67148:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 22 chunk: [22, 1048598) 1048598, 1048576 00020000:00200000:2.0:1775764862.630063:0:67148:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764862.630073:0:67148:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 256] 00000008:00010000:2.0:1775764862.630077:0:67148:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 1 00000008:00010000:2.0:1775764862.630079:0:67148:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8fb0, osc lock ffff8b238a3f8be0, flags 0x40000 00010000:00010000:2.0:1775764862.630085:0:67148:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764862.630091:0:67148:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1052671) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764862.630097:0:67148:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764862.630101:0:67148:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764862.630107:0:67148:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764862.630111:0:67148:0:(vvp_io.c:826:vvp_io_read_start()) f27.sanity-pcc: read [22, 1048598) 00000001:00010000:2.0:1775764862.630116:0:67148:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x1f:0x0] 00000008:00010000:2.0:1775764862.630118:0:67148:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764862.630119:0:67148:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8130, osc lock ffff8b238a3f8130, flags 0x400001002 00010000:00010000:2.0:1775764862.630122:0:67148:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764862.630127:0:67148:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:2.0:1775764862.630131:0:67148:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764862.630132:0:67148:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 8/1,1 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764862.630136:0:67148:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 7/1,0 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:2.0:1775764862.630139:0:67148:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764862.630141:0:67148:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1f:0x0] updating i_size 22 i_blocks 1 00000020:00010000:2.0:1775764862.630143:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027c6d8 (ffff8b23812ec638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764862.630144:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027c6d8 (ffff8b23812ec638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764862.630145:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8130 (ffff8b23812ec098/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764862.630147:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027c6d8 (ffff8b23812ec638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764862.630148:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8130 (ffff8b23812ec638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764862.630151:0:67148:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:22 00000020:00010000:2.0:1775764862.630154:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b238a36b308 (ffff8b23812ec638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764862.630156:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b238a36b308 (ffff8b23812ec638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764862.630157:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8fb0 (ffff8b23812ec098/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764862.630160:0:67148:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764862.630164:0:67148:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:2.0:1775764862.630169:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b238a36b308 (ffff8b23812ec638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764862.630170:0:67148:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8fb0 (ffff8b23812ec638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764862.630172:0:67148:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1775764862.630180:0:67148:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1f:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764862.630181:0:67148:0:(file.c:2206:ll_file_io_generic()) f27.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1775764862.630183:0:67148:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200000:2.0:1775764862.630404:0:67148:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1f:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764862.630410:0:67148:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:1.0:1775764862.632355:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119085184 00000100:00100000:1.0:1775764862.632365:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119085184 00000100:00100000:1.0:1775764862.632439:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 306 00000100:00100000:1.0:1775764862.632448:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daf480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42945:x1862022119085184:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764862.632515:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764862.632576:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764862.632580:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daf480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:42945:x1862022119085184:12345-192.168.202.128@tcp:104: Request processed in 134us (375us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764862.632585:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 306 00010000:00010000:2.0:1775764862.632628:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764862.632633:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764862.632639:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2380030200 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:2.0:1775764862.632644:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800428400000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00010000:2.0:1775764862.632690:0:46150:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 00000000a8e7d926 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800429400000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00010000:2.0:1775764862.632704:0:46150:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 00000000a8e7d926 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800429400000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764862.632760:0:46150:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 5/0,0 mode: PW/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800429400000000 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00100000:2.0:1775764862.632822:0:46150:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea5500 - object 0x240000400:287 offset 0<>22 00000080:00200000:2.0:1775764862.635343:0:46150:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764862.635400:0:46150:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775764862.635526:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764862.635532:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764862.635819:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764862.635829:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b2389398000 lock: 00000000b3d70b91/0x73cfd86cfc17c2c4 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e03052fc1 expref: -99 pid: 66748 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764862.635852:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764862.635854:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764862.635856:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764862.638061:0:67148:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17c2d9 lrc: 2/0,0 mode: PW/PW res: [0x200003ab4:0x1f:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x57b8ff0e03052ff2 expref: -99 pid: 67148 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764862.638076:0:67148:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fe00/0x73cfd86cfc17c2d9 lrc: 0/0,0 mode: --/PW res: [0x200003ab4:0x1f:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x57b8ff0e03052ff2 expref: -99 pid: 67148 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764862.638090:0:67148:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864f0e00 x1862022098712832/t34359739399(34359739399) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764878 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764862.638097:0:67148:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764862.638101:0:67148:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6680 x1862022098713600/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 504/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764862.638117:0:67148:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67148:1862022098713600:192.168.202.128@tcp:35:lfs.0 00000100:00100000:2.0:1775764862.638169:0:67148:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bcc0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764862.640646:0:46140:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1775764862.640667:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119085312 00000100:00100000:1.0:1775764862.640676:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119085312 00000100:00100000:1.0:1775764862.640726:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 307 00000100:00100000:1.0:1775764862.640733:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23b8702050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:47878:x1862022119085312:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764862.640749:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030000/0x73cfd86cfc17c2e0 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03053000 expref: -99 pid: 67148 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764862.640823:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764862.640829:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23b8702050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:47878:x1862022119085312:12345-192.168.202.128@tcp:104: Request processed in 97us (414us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764862.640836:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 307 00000100:00100000:2.0:1775764862.641051:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098713728:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764862.646130:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098713728:192.168.202.128@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764862.905083:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764863.031966:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2378:1862022098714624:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764863.033608:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764863.033629:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2378:1862022098714752:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764863.041650:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2378:1862022098714624:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764863.041692:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2378:1862022098714752:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764863.748441:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:2.0:1775764863.761115:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764863.761147:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098714880:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764863.771734:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098714880:192.168.202.128@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764863.928406:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764865.979776:0:67202:0:(ldlm_resource.c:195:lru_size_store()) dropping all unused locks from namespace lustre-MDT0000-mdc-ffff8b2389398000 00010000:00010000:1.0:1775764865.982170:0:67202:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 31/0 locks to regular blp list, 0 blwis in pool 00010000:00010000:2.0:1775764865.982608:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000002:00100000:2.0:1775764865.989854:0:46150:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea4e00 x1862022098658688/t34359739265(34359739265) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764814 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764865.989930:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c02b lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x10:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e030526d0 expref: -99 pid: 64938 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000002:00100000:2.0:1775764865.989990:0:46150:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea5180 x1862022098658944/t34359739266(34359739266) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764814 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764865.990000:0:46150:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764865.990005:0:46150:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4e00 x1862022098715392/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764865.990028:0:46150:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_02:fcf15866-4739-4595-8fe1-44abde7ee9d7:46150:1862022098715392:192.168.202.128@tcp:35:ldlm_bl.0 00000100:00100000:2.0:1775764865.990156:0:46150:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b540 going to sleep for 11 seconds 00000100:00100000:2.0:1775764865.999084:0:46150:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_02:fcf15866-4739-4595-8fe1-44abde7ee9d7:46150:1862022098715392:192.168.202.128@tcp:35:ldlm_bl.0 00000002:00100000:2.0:1775764865.999119:0:46150:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea5180 x1862022098658944/t34359739266(34359739266) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764814 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764865.999182:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c008 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052652 expref: -99 pid: 64892 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764865.999228:0:46150:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x10:0x0]: losing layout lock 00010000:00010000:2.0:1775764865.999257:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17c01d lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e030526a6 expref: -99 pid: 64922 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000002:00100000:2.0:1775764865.999312:0:46150:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b2382daea00 x1862022098658432/t34359739264(34359739264) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764814 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cat.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764865.999321:0:46150:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764865.999325:0:46150:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4e00 x1862022098715648/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764865.999347:0:46150:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_02:fcf15866-4739-4595-8fe1-44abde7ee9d7:46150:1862022098715648:192.168.202.128@tcp:35:ldlm_bl.0 00000100:00100000:2.0:1775764865.999416:0:46150:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b540 going to sleep for 11 seconds 00000100:00100000:2.0:1775764866.002389:0:46150:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_02:fcf15866-4739-4595-8fe1-44abde7ee9d7:46150:1862022098715648:192.168.202.128@tcp:35:ldlm_bl.0 00000002:00100000:2.0:1775764866.002426:0:46150:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b2382daea00 x1862022098658432/t34359739264(34359739264) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764814 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cat.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764866.002541:0:46150:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x10:0x0]: losing layout lock 00010000:00010000:2.0:1775764866.002591:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bff3 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0305260c expref: -99 pid: 64892 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764866.002612:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf83 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e030524a0 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764866.002623:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e800/0x73cfd86cfc17bf67 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052430 expref: -99 pid: 64101 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764866.002642:0:46150:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0xe:0x0]: losing layout lock 00010000:00010000:2.0:1775764866.002656:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf59 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0xe:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e030523ea expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764866.002667:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bf2f lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03052365 expref: -99 pid: 64056 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764866.002681:0:46150:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0xe:0x0]: losing layout lock 00010000:00010000:2.0:1775764866.002692:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bef7 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0305229a expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764866.002724:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17bf4b lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x20/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e030523b9 expref: -99 pid: 64079 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764866.002743:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17becd lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x11/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e0305221c expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764866.002754:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740e00/0x73cfd86cfc17beb1 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052197 expref: -99 pid: 63297 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000002:00100000:2.0:1775764866.002798:0:46150:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864ffb80 x1862022098630656/t34359739203(34359739203) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764780 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764866.002808:0:46150:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764866.002813:0:46150:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4e00 x1862022098715904/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764866.004265:0:46150:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_02:fcf15866-4739-4595-8fe1-44abde7ee9d7:46150:1862022098715904:192.168.202.128@tcp:35:ldlm_bl.0 00000100:00100000:2.0:1775764866.004380:0:46150:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b540 going to sleep for 11 seconds 00000100:00100000:2.0:1775764866.015389:0:46150:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_02:fcf15866-4739-4595-8fe1-44abde7ee9d7:46150:1862022098715904:192.168.202.128@tcp:35:ldlm_bl.0 00000002:00100000:2.0:1775764866.015432:0:46150:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864ffb80 x1862022098630656/t34359739203(34359739203) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/608 e 0 to 0 dl 1775764780 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764866.015702:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17beaa lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0xc:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03052182 expref: -99 pid: 63286 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000002:00100000:2.0:1775764866.016620:0:46150:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fd500 x1862022098630272/t34359739202(34359739202) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764780 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.500' uid:500 gid:500 projid:0 00000020:00100000:2.0:1775764866.016639:0:46150:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764866.016645:0:46150:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4e00 x1862022098716160/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764866.016670:0:46150:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_02:fcf15866-4739-4595-8fe1-44abde7ee9d7:46150:1862022098716160:192.168.202.128@tcp:35:ldlm_bl.0 00000100:00100000:2.0:1775764866.017194:0:46150:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b540 going to sleep for 11 seconds 00000100:00100000:2.0:1775764866.034535:0:46150:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_02:fcf15866-4739-4595-8fe1-44abde7ee9d7:46150:1862022098716160:192.168.202.128@tcp:35:ldlm_bl.0 00000002:00100000:2.0:1775764866.034570:0:46150:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fd500 x1862022098630272/t34359739202(34359739202) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764780 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.500' uid:500 gid:500 projid:0 00010000:00010000:2.0:1775764866.034621:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764866.034667:0:46150:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0xc:0x0]: losing layout lock 00010000:00010000:2.0:1775764866.034697:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be9c lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e0305215f expref: -99 pid: 63270 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000002:00100000:2.0:1775764866.035128:0:46150:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864ff100 x1862022098629376/t34359739199(34359739199) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764779 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.500' uid:500 gid:500 projid:0 00000020:00100000:2.0:1775764866.035140:0:46150:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764866.035145:0:46150:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4e00 x1862022098716416/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764866.057110:0:46150:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_02:fcf15866-4739-4595-8fe1-44abde7ee9d7:46150:1862022098717056:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764866.057431:0:46150:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b540 going to sleep for 11 seconds 00000100:00100000:2.0:1775764866.061356:0:46150:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_02:fcf15866-4739-4595-8fe1-44abde7ee9d7:46150:1862022098717056:192.168.202.128@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764866.061386:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e000/0x73cfd86cfc17baf9 lrc: 0/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305144d expref: -99 pid: 61408 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764866.061560:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17c2ee lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03053046 expref: -99 pid: 67174 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764866.061571:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497400/0x73cfd86cfc17c2e7 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03053015 expref: -99 pid: 67148 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764866.061584:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c200 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x17:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052d36 expref: -99 pid: 66239 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764866.061596:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c1f9 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x17:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052d05 expref: -99 pid: 66222 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764866.061607:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030400/0x73cfd86cfc17c024 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0xf:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030526bb expref: -99 pid: 64932 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764866.061616:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c02b lrc: 0/0,0 mode: --/CW res: [0x200003ab4:0x10:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030526d0 expref: -99 pid: 64938 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764866.061624:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c008 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052652 expref: -99 pid: 64892 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764866.061633:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497800/0x73cfd86cfc17c01d lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x10:0x0].0x0 bits 0xd/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030526a6 expref: -99 pid: 64922 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764866.061648:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ea00/0x73cfd86cfc17bff3 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305260c expref: -99 pid: 64892 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764866.061655:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebc00/0x73cfd86cfc17bf83 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0xf:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030524a0 expref: -99 pid: 64571 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764866.061665:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e800/0x73cfd86cfc17bf67 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052430 expref: -99 pid: 64101 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764866.061673:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eac00/0x73cfd86cfc17bf59 lrc: 0/0,0 mode: --/CW res: [0x200003ab4:0xe:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030523ea expref: -99 pid: 64085 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764866.061681:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045600/0x73cfd86cfc17bf2f lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0xe:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03052365 expref: -99 pid: 64056 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764866.061690:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044600/0x73cfd86cfc17bef7 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305229a expref: -99 pid: 64033 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764866.061697:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17bf4b lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0xe:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030523b9 expref: -99 pid: 64079 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764866.061706:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eaa00/0x73cfd86cfc17becd lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0xd:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e0305221c expref: -99 pid: 64014 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764866.061716:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740e00/0x73cfd86cfc17beb1 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0xb:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052197 expref: -99 pid: 63297 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764866.061729:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb400/0x73cfd86cfc17beaa lrc: 0/0,0 mode: --/CW res: [0x200003ab4:0xc:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052182 expref: -99 pid: 63286 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764866.061938:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ebe00/0x73cfd86cfc17be72 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030520da expref: -99 pid: 63191 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764866.061948:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17be9c lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0xc:0x0].0x0 bits 0xd/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e0305215f expref: -99 pid: 63270 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764866.061959:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eba00/0x73cfd86cfc17be64 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030520b0 expref: -99 pid: 63191 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764866.061966:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2044000/0x73cfd86cfc17be4f lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0xb:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03052078 expref: -99 pid: 63185 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764866.061975:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045000/0x73cfd86cfc17be2c lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051fe5 expref: -99 pid: 62468 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764866.061982:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740400/0x73cfd86cfc17be25 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051fd7 expref: -99 pid: 62468 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764866.061990:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6eb600/0x73cfd86cfc17be10 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x9:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051f6e expref: -99 pid: 62449 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764866.062000:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045e00/0x73cfd86cfc17bd22 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x6:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051bb6 expref: -99 pid: 62023 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764866.062013:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f800/0x73cfd86cfc17bd1b lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x6:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051ba8 expref: -99 pid: 62016 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764866.062023:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea800/0x73cfd86cfc17bd0d lrc: 0/0,0 mode: --/CW res: [0x200003ab4:0x6:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051b85 expref: -99 pid: 61601 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764866.062031:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23ac6ea000/0x73cfd86cfc17bc11 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03051821 expref: -99 pid: 61515 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764866.062041:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23c2045200/0x73cfd86cfc17baf2 lrc: 0/0,0 mode: --/PR res: [0x200000002:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03051431 expref: -99 pid: 61408 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764866.062112:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764866.062115:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764866.062117:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764866.064596:0:67202:0:(ldlm_resource.c:195:lru_size_store()) dropping all unused locks from namespace lustre-MDT0000-mdc-ffff8b238a3bf000 00000080:00200000:1.0:1775764866.376773:0:67219:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764866.376800:0:67219:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764866.376818:0:67219:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c2f5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053069 expref: -99 pid: 67212 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764866.376846:0:67219:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c2f5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053069 expref: -99 pid: 67212 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764866.376855:0:67219:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764866.376863:0:67219:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764866.376866:0:67219:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c2f5 00000080:00010000:1.0:1775764866.376873:0:67219:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c2f5 from it ffffa60b02267bf8 00010000:00010000:1.0:1775764866.376877:0:67219:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c2f5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053069 expref: -99 pid: 67212 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764866.376886:0:67219:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c2f5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053069 expref: -99 pid: 67212 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764866.376902:0:67219:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764866.376919:0:67219:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f27.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b23a00a7080, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1775764866.376925:0:67219:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f27.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open 00000002:00010000:1.0:1775764866.376939:0:67219:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f27.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100001 00010000:00010000:1.0:1775764866.376989:0:67219:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c2fc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67219 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764866.376996:0:67219:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c2fc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67219 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764866.377007:0:67219:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764866.377020:0:67219:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fdc00 x1862022098717568/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764866.377029:0:67219:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c2fc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67219 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764866.377050:0:67219:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:67219:1862022098717568:192.168.202.128@tcp:101:cat.0 00000100:00100000:1.0:1775764866.377620:0:67219:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b6543c0 going to sleep for 11 seconds 00000080:00200000:2.0:1775764866.501372:0:67229:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764866.501405:0:67229:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764866.501426:0:67229:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c2f5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053069 expref: -99 pid: 67212 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764866.501446:0:67229:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c2f5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053069 expref: -99 pid: 67212 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764866.501486:0:67229:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764866.501495:0:67229:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764866.501498:0:67229:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c2f5 00000080:00010000:2.0:1775764866.501506:0:67229:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c2f5 from it ffffa60b0223fbb0 00010000:00010000:2.0:1775764866.501511:0:67229:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c2f5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053069 expref: -99 pid: 67212 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764866.501519:0:67229:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c2f5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053069 expref: -99 pid: 67212 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764866.501536:0:67229:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764866.501545:0:67229:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764866.501574:0:67229:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffc964f7cd0 00000080:00200008:2.0:1775764866.501606:0:67229:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200008:2.0:1775764866.575134:0:67235:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764866.575148:0:67235:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764866.575170:0:67235:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764866.575189:0:67235:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c2f5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053069 expref: -99 pid: 67212 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764866.575205:0:67235:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c2f5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053069 expref: -99 pid: 67212 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764866.575213:0:67235:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764866.575222:0:67235:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764866.575226:0:67235:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c2f5 00000080:00010000:2.0:1775764866.575233:0:67235:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c2f5 from it ffffa60b02287cb8 00010000:00010000:2.0:1775764866.575236:0:67235:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c2f5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053069 expref: -99 pid: 67212 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764866.575244:0:67235:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c2f5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053069 expref: -99 pid: 67212 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764866.575275:0:67235:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764866.575281:0:67235:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764866.575300:0:67235:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c2f5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053069 expref: -99 pid: 67212 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764866.575309:0:67235:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c2f5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053069 expref: -99 pid: 67212 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764866.575314:0:67235:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764866.575319:0:67235:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764866.575321:0:67235:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c2f5 00000080:00010000:2.0:1775764866.575324:0:67235:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c2f5 from it ffffa60b02287b58 00010000:00010000:2.0:1775764866.575327:0:67235:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c2f5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053069 expref: -99 pid: 67212 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764866.575334:0:67235:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c2f5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053069 expref: -99 pid: 67212 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764866.575342:0:67235:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764866.575349:0:67235:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f27.sanity-pcc, flags=8 00000080:00200008:2.0:1775764866.575355:0:67235:0:(file.c:6226:ll_getattr_dentry()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764866.575359:0:67235:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88),name=f27.sanity-pcc 00000002:00010000:2.0:1775764866.575369:0:67235:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1f:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764866.575376:0:67235:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c2fc lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305309a expref: -99 pid: 67219 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764866.575388:0:67235:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c2fc lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305309a expref: -99 pid: 67219 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764866.575393:0:67235:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764866.575397:0:67235:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) 00000080:00010000:2.0:1775764866.575399:0:67235:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c2fc 00000080:00010000:2.0:1775764866.575402:0:67235:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c2fc from it ffffa60b02287cb8 00010000:00010000:2.0:1775764866.575405:0:67235:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c2fc lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305309a expref: -99 pid: 67219 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764866.575412:0:67235:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c2fc lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305309a expref: -99 pid: 67219 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764866.575439:0:67235:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764866.575445:0:67235:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764866.575572:0:67235:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c2f5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053069 expref: -99 pid: 67212 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000100:00100000:1.0:1775764866.896930:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119086976 00000100:00100000:1.0:1775764866.896942:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119086976 00000100:00100000:1.0:1775764866.897003:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 309 00000100:00100000:1.0:1775764866.897013:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dad180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42356:x1862022119086976:12345-192.168.202.128@tcp:107:hsm_cdtr.0 00000100:00100000:1.0:1775764866.897105:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dad180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42356:x1862022119086976:12345-192.168.202.128@tcp:107:hsm_cdtr.0 Request processed in 93us (444us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764866.897113:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 309 00000080:00010000:2.0:1775764866.971645:0:67236:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x22:0x0](ffff8b238990e3c8) for lock 0x73cfd86cfc17c31f 00000080:00010000:2.0:1775764866.971655:0:67236:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c31f from it ffffa60b02257c28 00010000:00010000:2.0:1775764866.971660:0:67236:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c31f lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x22:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053118 expref: -99 pid: 67236 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764866.971670:0:67236:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c31f lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x22:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053118 expref: -99 pid: 67236 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1775764866.971713:0:67236:0:(llite_lib.c:2274:ll_setattr_raw()) START file . :VOLATILE:0000:474F534F:fd=09:[0x200003ab4:0x22:0x0](ffff8b238990e3c8) current size 0, valid attrs 1846, mode 8b23, uid 0, gid 0, new size 0, atime 1775764866.000000000, mtime 1775764866.000000000, ctime 1775764866.000000000 00000020:00100000:2.0:1775764866.971764:0:67236:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764866.971771:0:67236:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5180 x1862022098720000/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764866.971796:0:67236:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67236:1862022098720000:192.168.202.128@tcp:36:lhsmtool_posix.0 00000100:00100000:2.0:1775764866.971883:0:67236:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764866.977069:0:67236:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67236:1862022098720000:192.168.202.128@tcp:36:lhsmtool_posix.0 00010000:00010000:2.0:1775764866.977129:0:67236:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 31 res 8589949620/34 (0 0) 00000080:00010000:2.0:1775764866.977142:0:67236:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x22:0x0](ffff8b238990e3c8) 00000002:00010000:2.0:1775764866.977153:0:67236:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x22:0x0]) in obj [0x200003ab4:0x22:0x0], intent: layout flags 0 00010000:00010000:2.0:1775764866.977356:0:67236:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67236 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764866.977367:0:67236:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67236 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764866.977379:0:67236:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67236 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764866.977402:0:67236:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67236:1862022098720128:192.168.202.128@tcp:101:lhsmtool_posix.0 00000100:00100000:2.0:1775764866.977514:0:67236:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764866.985996:0:67236:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67236:1862022098720128:192.168.202.128@tcp:101:lhsmtool_posix.0 00010000:00010000:2.0:1775764866.986031:0:67236:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764866.986047:0:67236:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764866.986061:0:67236:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea6300 x1862022098720128/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 376/400 e 0 to 0 dl 1775764882 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764866.986072:0:67236:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764866.986083:0:67236:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764866.986098:0:67236:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/34 (0 0) 00000080:00200000:2.0:1775764866.986103:0:67236:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x22:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764866.986122:0:67236:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x22:0x0](ffff8b238990e3c8) for lock 0x73cfd86cfc17c326 00000080:00010000:2.0:1775764866.986130:0:67236:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c326 from it ffffa60b02257a68 00010000:00010000:2.0:1775764866.986134:0:67236:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764866.986143:0:67236:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764866.986165:0:67236:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764866.986189:0:67236:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764866.986299:0:67236:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x22:0x0](0000000077664329) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764866.986376:0:67236:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x22:0x0]: layout version change: 4294967294 -> 0 00010000:00010000:2.0:1775764866.986397:0:67236:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764866.986405:0:67236:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764866.986441:0:67236:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x22:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764866.986490:0:67236:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:2.0:1775764866.986610:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764866.986629:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098720256:192.168.202.128@tcp:2:lhsmtool_posix.0 00000100:00100000:2.0:1775764866.993881:0:2378:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b2389398000: committing for last_committed 34359738470 gen 1 00000100:00100000:2.0:1775764866.993916:0:2378:0:(client.c:3126:ptlrpc_free_committed()) @@@ stopping search req@ffff8b23b9ea6300 x1862022098720256/t34359738473(34359738473) o2->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 440/432 e 0 to 0 dl 1775764882 ref 2 fl Rpc:RQU/604/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764866.993947:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098720256:192.168.202.128@tcp:2:lhsmtool_posix.0 00010000:00010000:2.0:1775764866.999561:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764867.026499:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119087104 00000100:00100000:1.0:1775764867.026511:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119087104 00000100:00100000:1.0:1775764867.026573:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 310 00000100:00100000:1.0:1775764867.026586:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42944:x1862022119087104:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764867.026605:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000887119ad/0x73cfd86cfc17c334 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e0305313b expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764867.026694:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764867.026700:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42944:x1862022119087104:12345-192.168.202.128@tcp:104: Request processed in 118us (387us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764867.026707:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 310 00010000:00010000:2.0:1775764867.026884:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764867.026891:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000887119ad/0x73cfd86cfc17c334 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x57b8ff0e0305313b expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764867.026900:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5f800 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:2.0:1775764867.026906:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000887119ad/0x73cfd86cfc17c334 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x57b8ff0e0305313b expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764867.026961:0:46150:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x22:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764867.026977:0:46150:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x22:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775764867.027061:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000887119ad/0x73cfd86cfc17c334 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e0305313b expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764867.027069:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764867.027109:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000887119ad/0x73cfd86cfc17c334 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e0305313b expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764867.027118:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000887119ad/0x73cfd86cfc17c334 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x57b8ff0e0305313b expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764867.027137:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764867.027139:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764867.027140:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764867.027149:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1775764867.027163:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098721536:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764867.030380:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764867.030409:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119087232 00000100:00100000:1.0:1775764867.030415:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119087232 00000100:00100000:1.0:1775764867.030498:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 311 00000100:00100000:1.0:1775764867.030508:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23864fca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42309:x1862022119087232:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764867.030520:0:46140:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000032f85aa7/0x73cfd86cfc17c33b lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03053142 expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764867.030530:0:46140:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000032f85aa7/0x73cfd86cfc17c33b lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x20000 nid: local remote: 0x57b8ff0e03053142 expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764867.030540:0:46140:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000032f85aa7/0x73cfd86cfc17c33b lrc: 5/0,1 mode: --/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000020000 nid: local remote: 0x57b8ff0e03053142 expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764867.030549:0:46140:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000032f85aa7/0x73cfd86cfc17c33b lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x57b8ff0e03053142 expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764867.030558:0:46140:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000032f85aa7/0x73cfd86cfc17c33b lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10080000020000 nid: local remote: 0x57b8ff0e03053142 expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764867.030567:0:46140:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000032f85aa7/0x73cfd86cfc17c33b lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x80000020000 nid: local remote: 0x57b8ff0e03053142 expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764867.030588:0:46140:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b23b9c5fc00) 00000100:00100000:1.0:1775764867.030648:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23864fca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:42309:x1862022119087232:12345-192.168.202.128@tcp:105: Request processed in 142us (276us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764867.030656:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 311 00000080:00200000:2.0:1775764867.032444:0:67236:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x22:0x0] updating i_size 22 i_blocks 1 00000080:00200000:2.0:1775764867.032483:0:67236:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 22 00000080:00200000:2.0:1775764867.032702:0:67236:0:(file.c:5304:ll_fsync()) VFS Op:inode=[0x200003ab4:0x22:0x0](ffff8b238990e3c8), start 0, end 9223372036854775807, datasync 0 00000080:00200000:2.0:1775764867.032736:0:67236:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x22:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764867.032757:0:67236:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:2.0:1775764867.032791:0:67236:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 000000007376cb7b ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000032f85aa7/0x73cfd86cfc17c33b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03053142 expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1775764867.032813:0:67236:0:(osc_cache.c:1167:osc_extent_make_ready()) ### extent: 000000007376cb7b ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000032f85aa7/0x73cfd86cfc17c33b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03053142 expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764867.032859:0:67236:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000032f85aa7/0x73cfd86cfc17c33b lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000020000 nid: local remote: 0x57b8ff0e03053142 expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00100000:2.0:1775764867.032928:0:67236:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea7800 - object 0x280000400:287 offset 0<>22 00000080:00200000:2.0:1775764867.037592:0:67236:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x22:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:2.0:1775764867.037661:0:67236:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67236:1862022098721792:192.168.202.128@tcp:44:lhsmtool_posix.0 00000100:00100000:2.0:1775764867.037733:0:67236:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bcc0 going to sleep for 11 seconds 00000080:00200000:2.0:1775764867.142986:0:67236:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x22:0x0] updating i_size 22 i_blocks 11 00000080:00200000:2.0:1775764867.143032:0:67236:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x22:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:2.0:1775764867.143065:0:67236:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x22:0x0](ffff8b238990e3c8) cmd=c03066b0 arg=7fdbee2c2ac0 00000080:00200008:2.0:1775764867.143077:0:67236:0:(llite_lib.c:2274:ll_setattr_raw()) START file . :VOLATILE:0000:474F534F:fd=09:[0x200003ab4:0x22:0x0](ffff8b238990e3c8) current size 22, valid attrs 1f0, mode 0, uid 0, gid 0, new size 0, atime 1775764862.000000000, mtime 1775764862.000000000, ctime 1775764862.000000000 00010000:00010000:2.0:1775764867.143117:0:67236:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17c32d lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x22:0x0].0x0 bits 0x1a/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03053134 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764867.143145:0:67236:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764867.143207:0:67236:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17c32d lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x22:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03053134 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764867.143214:0:67236:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764867.143218:0:67236:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17c32d lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x22:0x0].0x0 bits 0x1a/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03053134 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000020:00100000:2.0:1775764867.143236:0:67236:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1775764867.143245:0:67236:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5f80 x1862022098722176/t0(0) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764867.143268:0:67236:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67236:1862022098722176:192.168.202.128@tcp:36:lhsmtool_posix.0 00000100:00100000:2.0:1775764867.143439:0:67236:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764867.152855:0:67236:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 34359739424 gen 1 00000100:00100000:2.0:1775764867.152876:0:67236:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b2382daed80 x1862022098719232/t34359739422(34359739422) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 608/664 e 0 to 0 dl 1775764882 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764867.152901:0:67236:0:(client.c:3126:ptlrpc_free_committed()) @@@ stopping search req@ffff8b23b9ea5f80 x1862022098722176/t34359739425(34359739425) o36->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 488/456 e 0 to 0 dl 1775764883 ref 3 fl Rpc:RQU/204/0 rc 0/0 job:'lhsmtool_posix.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764867.152914:0:67236:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67236:1862022098722176:192.168.202.128@tcp:36:lhsmtool_posix.0 00000080:00200000:2.0:1775764867.153019:0:67236:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x22:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764867.153034:0:67236:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764867.157094:0:67236:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x22:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00800000:00000080:2.0:1775764867.157120:0:67236:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d4 len=384 karg=ffff8b23ac7e1600 uarg=0000000000000000 00000002:00000080:2.0:1775764867.157131:0:67236:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d4 len=384 karg=ffff8b23ac7e1600 uarg=0000000000000000 00000020:00100000:2.0:1775764867.157160:0:67236:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 55, max 7 00000100:00100000:2.0:1775764867.157180:0:67236:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4e00 x1862022098722560/t0(0) o55->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 472/224 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764867.157203:0:67236:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67236:1862022098722560:192.168.202.128@tcp:55:lhsmtool_posix.0 00000100:00100000:2.0:1775764867.157276:0:67236:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764867.160506:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119087488 00000100:00100000:1.0:1775764867.160515:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119087488 00000100:00100000:1.0:1775764867.160609:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 312 00000100:00100000:1.0:1775764867.160620:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daf100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42325:x1862022119087488:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764867.160637:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c31f lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x22:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03053118 expref: -99 pid: 67236 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764867.160720:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764867.160726:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daf100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42325:x1862022119087488:12345-192.168.202.128@tcp:104: Request processed in 109us (339us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764867.160734:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 312 00010000:00010000:2.0:1775764867.160801:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764867.160808:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c31f lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x22:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e03053118 expref: -99 pid: 67236 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764867.160818:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5f200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764867.160823:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c31f lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x22:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03053118 expref: -99 pid: 67236 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764867.160833:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c31f lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x22:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0x57b8ff0e03053118 expref: -99 pid: 67236 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764867.160842:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c31f lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x22:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0x57b8ff0e03053118 expref: -99 pid: 67236 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764867.160892:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c31f lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x22:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03053118 expref: -99 pid: 67236 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764867.160899:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764867.160929:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c31f lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x22:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03053118 expref: -99 pid: 67236 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764867.160937:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5f200/0x73cfd86cfc17c31f lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x22:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0x57b8ff0e03053118 expref: -99 pid: 67236 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764867.160951:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764867.160953:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764867.160955:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764867.168905:0:67236:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67236:1862022098722560:192.168.202.128@tcp:55:lhsmtool_posix.0 00000080:00200008:2.0:1775764867.168944:0:67236:0:(llite_lib.c:2524:ll_setattr_raw()) COMPLETED file . :VOLATILE:0000:474F534F:fd=09:[0x200003ab4:0x22:0x0](ffff8b238990e3c8) current size 22, valid attrs 1f0, mode 0, uid 0, gid 0, new size 0, atime 1775764862.000000000, mtime 1775764862.000000000, ctime 1775764862.000000000, rc 0 00000080:00200080:2.0:1775764867.168968:0:67236:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=400866d7 arg=7fdbe8000b78 00000080:00200000:2.0:1775764867.169017:0:67236:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x22:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764867.169031:0:67236:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:1.0:1775764867.171972:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764867.172397:0:46150:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764867.172403:0:46150:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000032f85aa7/0x73cfd86cfc17c33b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800420000020000 nid: local remote: 0x57b8ff0e03053142 expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764867.172411:0:46150:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5fc00 already unused, calling callback (ffffffffc10982c0) 00010000:00010000:2.0:1775764867.172416:0:46150:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000032f85aa7/0x73cfd86cfc17c33b lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800428400020000 nid: local remote: 0x57b8ff0e03053142 expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764867.172515:0:46150:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x22:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764867.172565:0:46150:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x22:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775764867.172648:0:46150:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000032f85aa7/0x73cfd86cfc17c33b lrc: 2/0,0 mode: --/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e03053142 expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764867.172656:0:46150:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764867.172684:0:46150:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000032f85aa7/0x73cfd86cfc17c33b lrc: 1/0,0 mode: --/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e03053142 expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764867.172693:0:46150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000032f85aa7/0x73cfd86cfc17c33b lrc: 0/0,0 mode: --/PW res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804c69400020020 nid: local remote: 0x57b8ff0e03053142 expref: -99 pid: 67236 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764867.172720:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764867.172723:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764867.172724:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00800000:00000080:2.0:1775764867.177924:0:67236:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b02257cf0 uarg=0000000000000000 00000002:00000080:2.0:1775764867.177938:0:67236:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d8 len=64 karg=ffffa60b02257cf0 uarg=0000000000000000 00000020:00100000:2.0:1775764867.177968:0:67236:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 57, max 7 00000100:00100000:2.0:1775764867.177976:0:67236:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5c00 x1862022098723200/t0(0) o57->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764867.178002:0:67236:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67236:1862022098723200:192.168.202.128@tcp:57:lhsmtool_posix.0 00000100:00100000:2.0:1775764867.178078:0:67236:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b240 going to sleep for 11 seconds 00000100:00100000:1.0:1775764867.185613:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119087744 00000100:00100000:1.0:1775764867.185715:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119087744 00000100:00100000:1.0:1775764867.185940:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 314 00000100:00100000:1.0:1775764867.185952:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dad180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119087744:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764867.185972:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764867.186436:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764867.186445:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dad180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119087744:12345-192.168.202.128@tcp:104: Request processed in 495us (1293us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764867.186479:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 314 00010000:00010000:2.0:1775764867.188470:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764867.188484:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764867.188499:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2380030200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764867.188505:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764867.188515:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764867.188527:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 3/0,0 mode: CR/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764867.188595:0:46149:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x22:0x0]: losing layout lock 00010000:00010000:2.0:1775764867.188920:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 2/0,0 mode: --/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764867.188930:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764867.189152:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764867.189178:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c326 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x22:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03053126 expref: -99 pid: 67236 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764867.189203:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764867.189205:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764867.189207:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764867.202100:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119088128 00000100:00100000:1.0:1775764867.202111:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119088128 00000100:00100000:1.0:1775764867.202182:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 315 00000100:00100000:1.0:1775764867.202192:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119088128:12345-192.168.202.128@tcp:105: 00010000:00010000:1.0:1775764867.202210:0:46139:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c303 lrc: 5/1,0 mode: --/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030530af expref: -99 pid: 67235 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764867.202230:0:46139:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c303 lrc: 5/1,0 mode: --/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030530af expref: -99 pid: 67235 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764867.202240:0:46139:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c303 lrc: 5/1,0 mode: --/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030530af expref: -99 pid: 67235 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764867.202250:0:46139:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c303 lrc: 5/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e030530af expref: -99 pid: 67235 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764867.202260:0:46139:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c303 lrc: 6/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10080000000000 nid: local remote: 0x57b8ff0e030530af expref: -99 pid: 67235 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764867.202271:0:46139:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c303 lrc: 6/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x80000000000 nid: local remote: 0x57b8ff0e030530af expref: -99 pid: 67235 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764867.202313:0:46139:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff8b23b9c5e600) 00000100:00100000:1.0:1775764867.202395:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119088128:12345-192.168.202.128@tcp:105: Request processed in 204us (582us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764867.202404:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 315 00010000:00010000:2.0:1775764867.220736:0:67235:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c303 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030530af expref: -99 pid: 67235 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764867.220767:0:67235:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c303 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030530af expref: -99 pid: 67235 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764867.220807:0:67235:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764867.220817:0:67235:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764867.220857:0:67235:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c303 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030530af expref: -99 pid: 67235 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764867.220870:0:67235:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c303 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030530af expref: -99 pid: 67235 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764867.220879:0:67235:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1f:0x0](000000000e8c9725) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c303 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030530af expref: -99 pid: 67235 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764867.220918:0:67235:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764867.220922:0:67235:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764867.220974:0:67235:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1f:0x0]: layout version change: 4294967294 -> 2 00010000:00010000:2.0:1775764867.220985:0:67235:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c303 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030530af expref: -99 pid: 67235 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764867.220994:0:67235:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c303 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030530af expref: -99 pid: 67235 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00800000:00000080:2.0:1775764867.221006:0:67235:0:(lmv_obd.c:907:lmv_iocontrol()) lustre-clilmv-ffff8b2389398000: cmd=401866d9 len=56 karg=ffff8b238872ea80 uarg=0000000000000000 00000002:00000080:2.0:1775764867.221016:0:67235:0:(mdc_request.c:2304:mdc_iocontrol()) lustre-MDT0000-mdc-ffff8b2389398000: cmd=401866d9 len=56 karg=ffff8b238872ea80 uarg=0000000000000000 00000020:00100000:2.0:1775764867.221048:0:67235:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 2 is allocated opc 58, max 7 00000100:00100000:2.0:1775764867.221060:0:67235:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6680 x1862022098723968/t0(0) o58->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 512/224 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764867.221087:0:67235:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67235:1862022098723968:192.168.202.128@tcp:58:lfs.0 00000100:00100000:2.0:1775764867.221345:0:67235:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764867.228064:0:67235:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67235:1862022098723968:192.168.202.128@tcp:58:lfs.0 00000080:00200008:2.0:1775764867.228109:0:67235:0:(file.c:447:ll_file_release()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), flags 154002 00000080:00200000:1.0:1775764867.279816:0:67238:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764867.279848:0:67238:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764867.279915:0:67238:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17c342 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67238 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764867.279925:0:67238:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17c342 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67238 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764867.279939:0:67238:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17c342 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67238 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764867.279963:0:67238:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67238:1862022098724096:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764867.280269:0:67238:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b6546c0 going to sleep for 11 seconds 00010000:00010000:2.0:1775764868.023631:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764868.263925:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:1.0:1775764868.263966:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119088256 00000100:00100000:1.0:1775764868.263976:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119088256 00000100:00100000:1.0:1775764868.264024:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 316 00000100:00100000:1.0:1775764868.264036:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b23864fca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42356:x1862022119088256:12345-192.168.202.128@tcp:107:hsm_cdtr.0 00000100:00100000:1.0:1775764868.264121:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b23864fca80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42356:x1862022119088256:12345-192.168.202.128@tcp:107:hsm_cdtr.0 Request processed in 89us (207us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764868.264127:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 316 00010000:00010000:2.0:1775764868.292999:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764868.293008:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c350 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e030531ab expref: -99 pid: 67251 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764868.293018:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5e200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764868.293023:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c350 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e030531ab expref: -99 pid: 67251 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764868.293031:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c350 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e030531ab expref: -99 pid: 67251 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764868.293048:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c350 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e030531ab expref: -99 pid: 67251 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764868.301257:0:67251:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:67251:1862022098725760:192.168.202.128@tcp:35:lhsmtool_posix.0 00000002:00100000:2.0:1775764868.301304:0:67251:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864f2680 x1862022098725248/t34359739434(34359739434) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764884 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764869.047638:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1775764869.179097:0:8:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764869.179117:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764869.179188:0:8:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:1.0:1775764870.610914:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098726272:192.168.202.128@tcp:101:lfs.0 00000080:00200000:2.0:1775764870.618082:0:67288:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1f:0x0] updating i_size 22 i_blocks 11 00000020:00010000:2.0:1775764870.618088:0:67288:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2398371488 (ffff8b2383fb5c68/0) at cl_lock_release():245 00000020:00010000:2.0:1775764870.618091:0:67288:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2398371488 (ffff8b2383fb5c68/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764870.618096:0:67288:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23849865b0 (ffff8b2383fb5368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764870.618100:0:67288:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2398371488 (ffff8b2383fb5c68/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764870.618102:0:67288:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23849865b0 (ffff8b2383fb5c68/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764870.618140:0:67288:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1f:0x0] ignore/verify layout 0/1, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764870.618193:0:67288:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764870.618211:0:67288:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764870.618225:0:67288:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17c342 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053181 expref: -99 pid: 67238 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764870.618235:0:67288:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17c342 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053181 expref: -99 pid: 67238 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764870.618246:0:67288:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764870.618263:0:67288:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764870.618266:0:67288:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c342 00000080:00010000:2.0:1775764870.618272:0:67288:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c342 from it ffffa60b02307bf8 00010000:00010000:2.0:1775764870.618274:0:67288:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17c342 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053181 expref: -99 pid: 67238 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764870.618281:0:67288:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17c342 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053181 expref: -99 pid: 67238 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764870.618290:0:67288:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764870.618295:0:67288:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f27.sanity-pcc, flags=265 00000080:00200000:2.0:1775764870.618300:0:67288:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), inode mode 81a4 mask 44 00000080:00200008:2.0:1775764870.618308:0:67288:0:(file.c:1057:ll_file_open()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), flags 100000 00000002:00010000:2.0:1775764870.618320:0:67288:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1f:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 04240100001 00010000:00010000:2.0:1775764870.618368:0:67288:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c35e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67288 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764870.618374:0:67288:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c35e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67288 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764870.618384:0:67288:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764870.618402:0:67288:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea6d80 x1862022098726528/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764870.618411:0:67288:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c35e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67288 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764870.618432:0:67288:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67288:1862022098726528:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764870.618632:0:67288:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764870.630895:0:67288:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67288:1862022098726528:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764870.630934:0:67288:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c35e lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ce expref: -99 pid: 67288 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764870.630946:0:67288:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c35e lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ce expref: -99 pid: 67288 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764870.630959:0:67288:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c35e lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030531ce expref: -99 pid: 67288 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764870.630981:0:67288:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c35e lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ce expref: -99 pid: 67288 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764870.630995:0:67288:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23b9ea6d80 x1862022098726528/t34359739438(34359739438) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764886 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764870.631008:0:67288:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea6d80 x1862022098726528/t34359739438(34359739438) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764886 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764870.631021:0:67288:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c35e lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ce expref: -99 pid: 67288 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764870.631037:0:67288:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c35e lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ce expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764870.631053:0:67288:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/31 (0 0) 00000080:00200000:2.0:1775764870.631060:0:67288:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764870.631180:0:67288:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1f:0x0]: layout version change: 2 -> 2 00000080:00010000:2.0:1775764870.631192:0:67288:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c35e 00000080:00010000:2.0:1775764870.631198:0:67288:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c35e from it ffffa60b02307bf0 00010000:00010000:2.0:1775764870.631202:0:67288:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c35e lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030531ce expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764870.631212:0:67288:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c35e lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030531ce expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00100000:2.0:1775764870.631236:0:67288:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea6d80 x1862022098726528/t34359739438(34359739438) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764886 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764870.631322:0:67288:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) cmd=400866fc arg=7ffc74b8a0f8 00000080:00010000:2.0:1775764870.631356:0:67288:0:(file.c:6988:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) 00000002:00010000:2.0:1775764870.631365:0:67288:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1f:0x0]) in obj [0x200003ab4:0x1f:0x0], intent: layout flags 02 00010000:00010000:2.0:1775764870.631380:0:67288:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e800/0x73cfd86cfc17c349 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e0305319d expref: -99 pid: 67238 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764870.631402:0:67288:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1f:0x0]: losing layout lock 00010000:00010000:2.0:1775764870.631432:0:67288:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c35e lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e030531ce expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764870.631447:0:67288:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1f:0x0]: losing layout lock 00010000:00010000:2.0:1775764870.631519:0:67288:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c303 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e030530af expref: -99 pid: 67235 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764870.631540:0:67288:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1f:0x0]: losing layout lock 00010000:00010000:2.0:1775764870.631875:0:67288:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e800/0x73cfd86cfc17c349 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e0305319d expref: -99 pid: 67238 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764870.631883:0:67288:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c35e lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030531ce expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764870.631889:0:67288:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c303 lrc: 1/0,0 mode: --/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030530af expref: -99 pid: 67235 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764870.631894:0:67288:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775764870.631897:0:67288:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e800/0x73cfd86cfc17c349 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e0305319d expref: -99 pid: 67238 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764870.631908:0:67288:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c35e lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030531ce expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764870.631917:0:67288:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e600/0x73cfd86cfc17c303 lrc: 0/0,0 mode: --/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e030530af expref: -99 pid: 67235 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764870.631942:0:67288:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x1f:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67288 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764870.631950:0:67288:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67288 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764870.631970:0:67288:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764870.631976:0:67288:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5880 x1862022098726784/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 392/880 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764870.631988:0:67288:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 3/0,1 mode: --/EX res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67288 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764870.632011:0:67288:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67288:1862022098726784:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764870.632150:0:67288:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495bcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764870.637086:0:67288:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67288:1862022098726784:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764870.637139:0:67288:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 4/0,1 mode: --/EX res: [0x200003ab4:0x1f:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764870.637152:0:67288:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 4/0,1 mode: --/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764870.637164:0:67288:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764870.637176:0:67288:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 4/0,1 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764870.637189:0:67288:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764870.637198:0:67288:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 5/1,1 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764870.637205:0:67288:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764870.637222:0:67288:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b23b9ea5880 x1862022098726784/t34359739439(34359739439) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 392/600 e 0 to 0 dl 1775764886 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764870.637241:0:67288:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764870.637283:0:67288:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764870.637297:0:67288:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/31 (0 0) 00000080:00200000:2.0:1775764870.637303:0:67288:0:(namei.c:953:ll_intent_lock()) intent lock 1024 on i1 [0x200003ab4:0x1f:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1775764870.637311:0:67288:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c365 00000080:00010000:2.0:1775764870.637331:0:67288:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c365 from it ffffa60b02307c10 00010000:00010000:2.0:1775764870.637335:0:67288:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764870.637357:0:67288:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764870.637421:0:67288:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764870.637434:0:67288:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1775764870.638059:0:67288:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1f:0x0](000000000e8c9725) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764870.638157:0:67288:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764870.638166:0:67288:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764870.638196:0:67288:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764870.638207:0:67288:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764870.638260:0:67288:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1775764870.638288:0:67288:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764870.638297:0:67288:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 2/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1775764870.638304:0:67288:0:(file.c:6871:ll_layout_lock_set()) ### file [0x200003ab4:0x1f:0x0](000000000e8c9725) being reconfigured ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764870.638336:0:67288:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764870.638340:0:67288:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764870.638365:0:67288:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x1f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:2.0:1775764870.638644:0:67288:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 22 blocks 11 [cma]time: 1775764862 1775764862 1775764862 by oinfo size 22 blocks 11 [cma]time 1775764862 1775764862 1775764862 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000008491a074/0x73cfd86cfc17c357 lrc: 1/0,0 mode: PR/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030531c0 expref: -99 pid: 67288 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764870.638708:0:67288:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1f:0x0]: layout version change: 4294967294 -> 3 00010000:00010000:2.0:1775764870.638720:0:67288:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764870.638728:0:67288:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:2.0:1775764870.639214:0:67288:0:(file.c:2454:do_file_read_iter()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0], ppos: 0, count: 1048576 00000080:00200000:2.0:1775764870.639243:0:67288:0:(rw.c:1928:ll_readpage()) fast read pgno: 0 00000080:00200000:2.0:1775764870.639282:0:67288:0:(file.c:2058:ll_file_io_generic()) f27.sanity-pcc: read ppos: 0, bytes: 1048576 00000080:00200000:2.0:1775764870.639306:0:67288:0:(file.c:1815:ll_io_set_mirror()) f27.sanity-pcc: desiginated mirror: 0 00000020:00200000:2.0:1775764870.639313:0:67288:0:(cl_io.c:214:cl_io_rw_init()) header@000000000ca6206a[0x110, 1, [0x200003ab4:0x1f:0x0] hash] 00000020:00200000:2.0:1775764870.639315:0:67288:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 1048576) 0 0 00000080:00200000:2.0:1775764870.639318:0:67288:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x1f:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1775764870.639338:0:67288:0:(lov_io.c:458:lov_io_mirror_init()) [0x200003ab4:0x1f:0x0]: flr state: 9, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1775764870.639342:0:67288:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:2.0:1775764870.639349:0:67288:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:2.0:1775764870.639377:0:67288:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764870.640068:0:67288:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:2.0:1775764870.640085:0:67288:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764870.640090:0:67288:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c89b0, osc lock ffff8b238a3f8be0, flags 0x400000000000000 00010000:00010000:2.0:1775764870.640109:0:67288:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000008491a074/0x73cfd86cfc17c357 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030531c0 expref: -99 pid: 67288 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764870.640123:0:67288:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000008491a074/0x73cfd86cfc17c357 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030531c0 expref: -99 pid: 67288 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764870.640135:0:67288:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000008491a074/0x73cfd86cfc17c357 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030531c0 expref: -99 pid: 67288 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1775764870.640145:0:67288:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 22, setting rss=22; kms=22, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000008491a074/0x73cfd86cfc17c357 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030531c0 expref: -99 pid: 67288 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764870.640157:0:67288:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000008491a074/0x73cfd86cfc17c357 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030531c0 expref: -99 pid: 67288 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764870.640163:0:67288:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000008491a074/0x73cfd86cfc17c357 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030531c0 expref: -99 pid: 67288 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764870.640173:0:67288:0:(vvp_io.c:826:vvp_io_read_start()) f27.sanity-pcc: read [0, 1048576) 00000001:00010000:2.0:1775764870.640188:0:67288:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x1f:0x0] 00000008:00010000:2.0:1775764870.640193:0:67288:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1775764870.640195:0:67288:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a00c8a30, osc lock ffff8b238a3f8130, flags 0x400000400001002 00010000:00010000:2.0:1775764870.640202:0:67288:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000008491a074/0x73cfd86cfc17c357 lrc: 4/2,0 mode: PR/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030531c0 expref: -99 pid: 67288 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764870.640210:0:67288:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000008491a074/0x73cfd86cfc17c357 lrc: 4/2,0 mode: PR/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030531c0 expref: -99 pid: 67288 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1775764870.640218:0:67288:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1775764870.640221:0:67288:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000008491a074/0x73cfd86cfc17c357 lrc: 7/2,0 mode: PR/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030531c0 expref: -99 pid: 67288 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764870.640228:0:67288:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000008491a074/0x73cfd86cfc17c357 lrc: 6/1,0 mode: PR/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e030531c0 expref: -99 pid: 67288 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1775764870.640234:0:67288:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1775764870.640239:0:67288:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x1f:0x0] updating i_size 22 i_blocks 11 00000020:00010000:2.0:1775764870.640243:0:67288:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b239027d240 (ffff8b23812ec248/0) at cl_lock_release():245 00000020:00010000:2.0:1775764870.640245:0:67288:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b239027d240 (ffff8b23812ec248/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764870.640258:0:67288:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a00c8a30 (ffff8b23812ecc68/1) at cl_lock_cancel():142 00000020:00010000:2.0:1775764870.640262:0:67288:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b239027d240 (ffff8b23812ec248/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764870.640265:0:67288:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a00c8a30 (ffff8b23812ec248/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764870.640274:0:67288:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:0 00000008:00010000:2.0:1775764870.640330:0:67288:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b2383c9eb40 00000008:00010000:2.0:1775764870.640437:0:46654:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b2383c9eb40 00010000:00010000:2.0:1775764870.640507:0:46654:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b2389398000 lock: 000000008491a074/0x73cfd86cfc17c357 lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x11f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e030531c0 expref: -99 pid: 67288 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00100000:2.0:1775764870.640585:0:46654:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8b23b9ea7b80 - object 0x280000400:287 offset 0<>4096 00000100:00100000:2.0:1775764870.640680:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764870.640716:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098726912:192.168.202.128@tcp:3:lfs.0 00000008:00010000:2.0:1775764870.643265:0:2379:0:(osc_cache.c:901:osc_extent_finish()) ### no dlm lock: extent: ffff8b2383c9eb40 00000020:00200000:2.0:1775764870.643279:0:2379:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1775764870.643725:0:2379:0:(osc_cache.c:496:__osc_extent_remove()) ### no dlm lock: extent: ffff8b2383c9eb40 00000100:00100000:2.0:1775764870.643770:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098726912:192.168.202.128@tcp:3:lfs.0 00000080:00200000:2.0:1775764870.725754:0:67298:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764870.725784:0:67298:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764870.725801:0:67298:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17c342 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053181 expref: -99 pid: 67238 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764870.725818:0:67298:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17c342 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053181 expref: -99 pid: 67238 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764870.725827:0:67298:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764870.725835:0:67298:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764870.725840:0:67298:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c342 00000080:00010000:2.0:1775764870.725847:0:67298:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c342 from it ffffa60b02327bb0 00010000:00010000:2.0:1775764870.725852:0:67298:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17c342 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053181 expref: -99 pid: 67238 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764870.725862:0:67298:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740200/0x73cfd86cfc17c342 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053181 expref: -99 pid: 67238 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764870.725877:0:67298:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764870.725885:0:67298:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764870.725909:0:67298:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffc4358cf50 00000100:00100000:2.0:1775764870.725974:0:67298:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67298:1862022098727296:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764870.726137:0:67298:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b6c0 going to sleep for 11 seconds 00000100:00100000:2.0:1775764870.730850:0:67298:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67298:1862022098727296:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764870.730935:0:67298:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00010000:00010000:2.0:1775764871.095700:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764873.144805:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764873.274047:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2379:1862022098727680:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764873.274883:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098727936:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764873.274916:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098728192:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764873.289175:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2379:1862022098727680:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764873.289196:0:2379:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 34359739441 gen 1 00000100:00100000:2.0:1775764873.289245:0:2379:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23864f3100 x1862022098727552/t34359739441(34359739441) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764887 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764873.289260:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098727936:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764873.289271:0:2379:0:(client.c:3082:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b2389398000: committing for last_committed 34359738476 gen 1 00000100:00100000:2.0:1775764873.289279:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098728192:192.168.202.128@tcp:400:kworker.0 00000080:00200008:1.0:1775764874.127365:0:67342:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764874.127380:0:67342:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764874.127401:0:67342:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764874.127529:0:67342:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67342 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764874.127539:0:67342:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67342 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764874.127549:0:67342:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67342 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764874.127575:0:67342:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67342:1862022098728320:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764874.127686:0:67342:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654840 going to sleep for 11 seconds 00010000:00010000:2.0:1775764874.169732:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1775764874.361536:0:67359:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764874.361570:0:67359:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764874.361591:0:67359:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764874.361608:0:67359:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764874.361617:0:67359:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764874.361625:0:67359:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764874.361628:0:67359:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c373 00000080:00010000:2.0:1775764874.361635:0:67359:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c373 from it ffffa60b0239fbf8 00010000:00010000:2.0:1775764874.361640:0:67359:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764874.361649:0:67359:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1775764874.361664:0:67359:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764874.361672:0:67359:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f27.sanity-pcc, flags=265 00000080:00200000:2.0:1775764874.361677:0:67359:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), inode mode 81a4 mask 44 00000080:00200008:2.0:1775764874.361683:0:67359:0:(file.c:1057:ll_file_open()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), flags 100000 00000002:00010000:2.0:1775764874.361698:0:67359:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1f:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 04240100001 00010000:00010000:2.0:1775764874.361748:0:67359:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c37a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67359 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764874.361756:0:67359:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c37a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67359 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764874.361769:0:67359:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764874.361782:0:67359:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea5180 x1862022098728576/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764874.361792:0:67359:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c37a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67359 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764874.361816:0:67359:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:67359:1862022098728576:192.168.202.128@tcp:101:cat.0 00000100:00100000:2.0:1775764874.361999:0:67359:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495ba80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764874.374737:0:67359:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:67359:1862022098728576:192.168.202.128@tcp:101:cat.0 00010000:00010000:2.0:1775764874.374772:0:67359:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c37a lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305323e expref: -99 pid: 67359 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764874.374783:0:67359:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c37a lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305323e expref: -99 pid: 67359 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764874.374795:0:67359:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c37a lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305323e expref: -99 pid: 67359 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764874.374806:0:67359:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c37a lrc: 4/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305323e expref: -99 pid: 67359 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764874.374828:0:67359:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23b9ea5180 x1862022098728576/t34359739442(34359739442) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764890 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764874.374841:0:67359:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea5180 x1862022098728576/t34359739442(34359739442) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764890 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764874.374854:0:67359:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 256 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c37a lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305323e expref: -99 pid: 67359 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764874.374868:0:67359:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c37a lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305323e expref: -99 pid: 67359 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764874.374878:0:67359:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 16 res 8589949620/31 (0 0) 00000080:00200000:2.0:1775764874.374886:0:67359:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764874.374990:0:67359:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x1f:0x0]: layout version change: 3 -> 3 00000080:00010000:2.0:1775764874.375002:0:67359:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c37a 00000080:00010000:2.0:1775764874.375010:0:67359:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c37a from it ffffa60b0239fbf0 00010000:00010000:2.0:1775764874.375014:0:67359:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c37a lrc: 3/1,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305323e expref: -99 pid: 67359 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764874.375025:0:67359:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c37a lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305323e expref: -99 pid: 67359 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00100000:2.0:1775764874.375051:0:67359:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea5180 x1862022098728576/t34359739442(34359739442) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764890 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'cat.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1775764874.375182:0:67359:0:(file.c:6226:ll_getattr_dentry()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764874.375189:0:67359:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88),name=f27.sanity-pcc 00000002:00010000:2.0:1775764874.375210:0:67359:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1f:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764874.375234:0:67359:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c36c lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305321b expref: -99 pid: 67314 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764874.375246:0:67359:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c36c lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305321b expref: -99 pid: 67314 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764874.375253:0:67359:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764874.375259:0:67359:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) 00000080:00010000:2.0:1775764874.375263:0:67359:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c36c 00000080:00010000:2.0:1775764874.375267:0:67359:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c36c from it ffffa60b0239fcd8 00010000:00010000:2.0:1775764874.375270:0:67359:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c36c lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305321b expref: -99 pid: 67314 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764874.375277:0:67359:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c36c lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305321b expref: -99 pid: 67314 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:2.0:1775764874.375368:0:67359:0:(file.c:2454:do_file_read_iter()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0], ppos: 0, count: 4194304 00000080:00200008:2.0:1775764874.375485:0:67359:0:(file.c:2454:do_file_read_iter()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0], ppos: 22, count: 4194304 00000080:00200008:2.0:1775764874.375556:0:67359:0:(file.c:447:ll_file_release()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), flags 100000 00010000:00010000:2.0:1775764874.375570:0:67359:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c37a lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305323e expref: -99 pid: 67359 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764874.476955:0:67369:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764874.476986:0:67369:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764874.477006:0:67369:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764874.477022:0:67369:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764874.477030:0:67369:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764874.477038:0:67369:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764874.477042:0:67369:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c373 00000080:00010000:1.0:1775764874.477049:0:67369:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c373 from it ffffa60b023e7bb0 00010000:00010000:1.0:1775764874.477790:0:67369:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764874.477807:0:67369:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764874.477826:0:67369:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764874.477835:0:67369:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:1.0:1775764874.477869:0:67369:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffc9f557510 00000080:00200008:1.0:1775764874.477903:0:67369:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200008:2.0:1775764874.558763:0:67375:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764874.558777:0:67375:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764874.558799:0:67375:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764874.558819:0:67375:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764874.558839:0:67375:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764874.558848:0:67375:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764874.558856:0:67375:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764874.558859:0:67375:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c373 00000080:00010000:2.0:1775764874.558865:0:67375:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c373 from it ffffa60b023c7cb8 00010000:00010000:2.0:1775764874.558869:0:67375:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764874.558879:0:67375:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764874.558905:0:67375:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764874.558911:0:67375:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764874.558917:0:67375:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764874.558925:0:67375:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764874.558931:0:67375:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764874.558935:0:67375:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764874.558937:0:67375:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c373 00000080:00010000:2.0:1775764874.558940:0:67375:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c373 from it ffffa60b023c7b58 00010000:00010000:2.0:1775764874.558943:0:67375:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764874.558951:0:67375:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1775764874.558960:0:67375:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764874.558966:0:67375:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f27.sanity-pcc, flags=8 00000080:00200008:2.0:1775764874.558973:0:67375:0:(file.c:6226:ll_getattr_dentry()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764874.558977:0:67375:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88),name=f27.sanity-pcc 00000002:00010000:2.0:1775764874.558987:0:67375:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1f:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764874.558995:0:67375:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c36c lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305321b expref: -99 pid: 67314 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764874.559003:0:67375:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c36c lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305321b expref: -99 pid: 67314 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764874.559009:0:67375:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764874.559012:0:67375:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) 00000080:00010000:2.0:1775764874.559014:0:67375:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c36c 00000080:00010000:2.0:1775764874.559017:0:67375:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c36c from it ffffa60b023c7cb8 00010000:00010000:2.0:1775764874.559020:0:67375:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c36c lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305321b expref: -99 pid: 67314 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764874.559026:0:67375:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c36c lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305321b expref: -99 pid: 67314 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1775764874.559192:0:67375:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764874.559201:0:67375:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764874.559208:0:67375:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764874.559226:0:67375:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1775764874.559232:0:67375:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764874.559236:0:67375:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764874.559239:0:67375:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c373 00000080:00010000:2.0:1775764874.559242:0:67375:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c373 from it ffffa60b023c7bf8 00010000:00010000:2.0:1775764874.559245:0:67375:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764874.559252:0:67375:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:2.0:1775764874.559260:0:67375:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764874.559265:0:67375:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f27.sanity-pcc, flags=265 00000080:00200000:2.0:1775764874.559268:0:67375:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), inode mode 81a4 mask 46 00000080:00200008:2.0:1775764874.559274:0:67375:0:(file.c:1057:ll_file_open()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), flags 154402 00000080:00200080:2.0:1775764874.559302:0:67375:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) cmd=c00466fc arg=7fff9746565c 00000080:00200008:2.0:1775764874.559369:0:67375:0:(file.c:447:ll_file_release()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), flags 154002 00000080:00200000:1.0:1775764874.661041:0:67385:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764874.661084:0:67385:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764874.661110:0:67385:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764874.661131:0:67385:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:1.0:1775764874.661139:0:67385:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764874.661147:0:67385:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764874.661150:0:67385:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c373 00000080:00010000:1.0:1775764874.661159:0:67385:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c373 from it ffffa60b023efbb0 00010000:00010000:1.0:1775764874.661163:0:67385:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1775764874.661171:0:67385:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:1.0:1775764874.661187:0:67385:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764874.661194:0:67385:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:1.0:1775764874.661229:0:67385:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffe2d675a40 00000080:00200008:1.0:1775764874.661260:0:67385:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000100:00100000:2.0:1775764874.822366:0:67314:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:67314:1862022098728832:192.168.202.128@tcp:35:multiop.0 00000002:00100000:2.0:1775764874.822403:0:67314:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864f3100 x1862022098727552/t34359739441(34359739441) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 784/864 e 0 to 0 dl 1775764887 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1775764875.048929:0:67415:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764875.048964:0:67415:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764875.048984:0:67415:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764875.049002:0:67415:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:2.0:1775764875.049011:0:67415:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764875.049019:0:67415:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764875.049023:0:67415:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c373 00000080:00010000:2.0:1775764875.049030:0:67415:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c373 from it ffffa60b023bfbb0 00010000:00010000:2.0:1775764875.049034:0:67415:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764875.049044:0:67415:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741c00/0x73cfd86cfc17c373 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053230 expref: -99 pid: 67342 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:2.0:1775764875.049059:0:67415:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764875.049068:0:67415:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764875.049170:0:67415:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffe855c4f90 00000080:00200008:2.0:1775764875.049214:0:67415:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00010000:00010000:2.0:1775764875.191689:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764875.448137:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119090048 00000100:00100000:1.0:1775764875.448147:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119090048 00010000:00010000:2.0:1775764875.449448:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764875.449670:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c36c lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0305321b expref: -99 pid: 67314 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764875.449682:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b9c5e200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764875.449687:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c36c lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e0305321b expref: -99 pid: 67314 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764875.449701:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c36c lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e0305321b expref: -99 pid: 67314 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764875.449710:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c36c lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e0305321b expref: -99 pid: 67314 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764875.449741:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031a00/0x73cfd86cfc17c37a lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xd/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305323e expref: -99 pid: 67359 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764875.449753:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031200/0x73cfd86cfc17c365 lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x1f:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030531ea expref: -99 pid: 67288 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764875.449801:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c36c lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e0305321b expref: -99 pid: 67314 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764875.449808:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764875.450017:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c36c lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e0305321b expref: -99 pid: 67314 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764875.450025:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e200/0x73cfd86cfc17c36c lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e0305321b expref: -99 pid: 67314 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764875.450043:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764875.450045:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764875.450047:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764875.450067:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098729216:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764875.461576:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098729216:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764875.463125:0:67431:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:67431:1862022098729088:192.168.202.128@tcp:101:multiop.0 00010000:00010000:1.0:1775764875.463184:0:67431:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c381 lrc: 4/0,1 mode: --/CW res: [0x200003ab4:0x1f:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053253 expref: -99 pid: 67431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764875.463220:0:67431:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c381 lrc: 4/0,1 mode: --/CW res: [0x200003ab4:0x1f:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053253 expref: -99 pid: 67431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764875.463242:0:67431:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c381 lrc: 4/0,1 mode: CW/CW res: [0x200003ab4:0x1f:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053253 expref: -99 pid: 67431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764875.463255:0:67431:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c381 lrc: 4/0,1 mode: CW/CW res: [0x200003ab4:0x1f:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053253 expref: -99 pid: 67431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764875.463272:0:67431:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=200002b, status=0 req@ffff8b2382daed80 x1862022098729088/t34359739444(34359739444) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764891 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764875.463291:0:67431:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382daed80 x1862022098729088/t34359739444(34359739444) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/864 e 0 to 0 dl 1775764891 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764875.463308:0:67431:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c381 lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0x1f:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053253 expref: -99 pid: 67431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764875.463319:0:67431:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 8 res 8589949620/31 (0 0) 00000080:00200000:1.0:1775764875.463327:0:67431:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1775764875.463438:0:67431:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c381 00000080:00010000:1.0:1775764875.463446:0:67431:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c381 from it ffffa60b02477bf0 00010000:00010000:1.0:1775764875.463474:0:67431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c381 lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0x1f:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053253 expref: -99 pid: 67431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764875.463485:0:67431:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c381 lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x1f:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053253 expref: -99 pid: 67431 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1775764875.463510:0:67431:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382daed80 x1862022098729088/t34359739444(34359739444) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764891 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764876.215570:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764877.239676:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764878.264551:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764878.294758:0:67458:0:(ldlm_resource.c:195:lru_size_store()) dropping all unused locks from namespace lustre-MDT0000-mdc-ffff8b2389398000 00010000:00010000:2.0:1775764878.294838:0:67458:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 3/3 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1775764878.328254:0:67458:0:(ldlm_resource.c:195:lru_size_store()) dropping all unused locks from namespace lustre-MDT0000-mdc-ffff8b238a3bf000 00000100:00100000:2.0:1775764878.394285:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2379:1862022098729728:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764878.394381:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098729984:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764878.394422:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1775764878.394428:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2379:1862022098729856:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764878.394445:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098730112:192.168.202.128@tcp:400:kworker.0 00000080:00200000:2.0:1775764878.487701:0:67469:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764878.487735:0:67469:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764878.487800:0:67469:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764878.487808:0:67469:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67469 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764878.487816:0:67469:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67469 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764878.488392:0:67469:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67469:1862022098730240:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764878.489239:0:67469:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764878.499742:0:67469:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67469:1862022098730240:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764878.499772:0:67469:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764878.499785:0:67469:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764878.499797:0:67469:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764878.499809:0:67469:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764878.499823:0:67469:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764878.499839:0:67469:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764878.499846:0:67469:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764878.499858:0:67469:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea4380 x1862022098730240/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764894 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764878.499870:0:67469:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764878.499882:0:67469:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764878.499888:0:67469:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764878.499918:0:67469:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764878.499925:0:67469:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c388 00000080:00010000:2.0:1775764878.499930:0:67469:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764878.499933:0:67469:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c388 from it ffffa60b024dfbb0 00010000:00010000:2.0:1775764878.499936:0:67469:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764878.499944:0:67469:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764878.499979:0:67469:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764878.499994:0:67469:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:2.0:1775764878.500051:0:67469:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffd103f7560 00000100:00100000:2.0:1775764878.500113:0:67469:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67469:1862022098730368:192.168.202.128@tcp:34:lfs.0 00000100:00100000:2.0:1775764878.500189:0:67469:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764878.517691:0:67469:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:67469:1862022098730368:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764878.517764:0:67469:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:1.0:1775764878.673010:0:67476:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764878.673039:0:67476:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764878.673057:0:67476:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764878.673073:0:67476:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764878.673081:0:67476:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764878.673090:0:67476:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764878.673094:0:67476:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c388 00000080:00010000:1.0:1775764878.673101:0:67476:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c388 from it ffffa60b024ffbf8 00010000:00010000:1.0:1775764878.673105:0:67476:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764878.673113:0:67476:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764878.673129:0:67476:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764878.673145:0:67476:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f27.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b2383ec9200, open_flags 8000, mode 0 opened 0 00000080:00200000:1.0:1775764878.673152:0:67476:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f27.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open 00000002:00010000:1.0:1775764878.673166:0:67476:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f27.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100001 00010000:00010000:1.0:1775764878.673226:0:67476:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17c38f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67476 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764878.673234:0:67476:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17c38f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67476 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764878.673250:0:67476:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764878.673262:0:67476:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ff100 x1862022098730624/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764878.673271:0:67476:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17c38f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67476 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764878.673295:0:67476:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job cat:fcf15866-4739-4595-8fe1-44abde7ee9d7:67476:1862022098730624:192.168.202.128@tcp:101:cat.0 00000100:00100000:1.0:1775764878.674912:0:67476:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b6543c0 going to sleep for 11 seconds 00000080:00200008:1.0:1775764878.901434:0:67492:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764878.902556:0:67492:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764878.902583:0:67492:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764878.902605:0:67492:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764878.902621:0:67492:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764878.902629:0:67492:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764878.902639:0:67492:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764878.902643:0:67492:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c388 00000080:00010000:1.0:1775764878.902650:0:67492:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c388 from it ffffa60b02527cb8 00010000:00010000:1.0:1775764878.902654:0:67492:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764878.902663:0:67492:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764878.902710:0:67492:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764878.902717:0:67492:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764878.902722:0:67492:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764878.902730:0:67492:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764878.902736:0:67492:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764878.902739:0:67492:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764878.902741:0:67492:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c388 00000080:00010000:1.0:1775764878.902744:0:67492:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c388 from it ffffa60b02527b58 00010000:00010000:1.0:1775764878.902747:0:67492:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764878.902754:0:67492:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764878.902760:0:67492:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764878.902766:0:67492:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f27.sanity-pcc, flags=8 00000080:00200008:1.0:1775764878.902774:0:67492:0:(file.c:6226:ll_getattr_dentry()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764878.902777:0:67492:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88),name=f27.sanity-pcc 00000002:00010000:1.0:1775764878.902786:0:67492:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x1f:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764878.902792:0:67492:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17c38f lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053299 expref: -99 pid: 67476 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764878.902805:0:67492:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17c38f lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053299 expref: -99 pid: 67476 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764878.902812:0:67492:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764878.902815:0:67492:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) 00000080:00010000:1.0:1775764878.902817:0:67492:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) for lock 0x73cfd86cfc17c38f 00000080:00010000:1.0:1775764878.902821:0:67492:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c38f from it ffffa60b02527cb8 00010000:00010000:1.0:1775764878.902823:0:67492:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17c38f lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03053299 expref: -99 pid: 67476 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764878.902830:0:67492:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17c38f lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053299 expref: -99 pid: 67476 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764878.903557:0:67492:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764878.903572:0:67492:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764878.903581:0:67492:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764878.903591:0:67492:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764878.903597:0:67492:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764878.903602:0:67492:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764878.903604:0:67492:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c388 00000080:00010000:1.0:1775764878.903608:0:67492:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c388 from it ffffa60b02527bf8 00010000:00010000:1.0:1775764878.903611:0:67492:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764878.903618:0:67492:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764878.903628:0:67492:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764878.903633:0:67492:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f27.sanity-pcc, flags=265 00000080:00200000:1.0:1775764878.903638:0:67492:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), inode mode 81a4 mask 46 00000080:00200008:1.0:1775764878.903646:0:67492:0:(file.c:1057:ll_file_open()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), flags 154402 00000080:00200080:1.0:1775764878.903685:0:67492:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88) cmd=c00466fc arg=7ffca846710c 00000080:00200008:1.0:1775764878.903753:0:67492:0:(file.c:447:ll_file_release()) START file f27.sanity-pcc:[0x200003ab4:0x1f:0x0](ffff8b2390ec5b88), flags 154002 00000080:00200000:1.0:1775764879.009133:0:67502:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764879.009162:0:67502:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764879.009694:0:67502:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764879.009759:0:67502:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1775764879.009769:0:67502:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764879.009778:0:67502:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764879.009782:0:67502:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c388 00000080:00010000:1.0:1775764879.009789:0:67502:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c388 from it ffffa60b02537bb0 00010000:00010000:1.0:1775764879.009793:0:67502:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1775764879.009801:0:67502:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1775764879.009824:0:67502:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764879.009833:0:67502:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:1.0:1775764879.009867:0:67502:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffccfffb7f0 00000080:00200008:1.0:1775764879.009903:0:67502:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000100:00100000:1.0:1775764879.122789:0:67431:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:67431:1862022098731136:192.168.202.128@tcp:35:multiop.0 00000002:00100000:1.0:1775764879.122814:0:67431:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b2382daed80 x1862022098729088/t34359739444(34359739444) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 776/864 e 0 to 0 dl 1775764891 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764883.511704:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2378:1862022098731392:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764883.511795:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098731648:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764883.511825:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:2.0:1775764883.511830:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2378:1862022098731264:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764883.511848:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098731520:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764883.511864:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098731776:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764883.514107:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2378:1862022098731264:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764883.514124:0:2378:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 34359739448 gen 1 00000100:00100000:2.0:1775764883.514142:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098731520:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764883.514155:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098731776:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764883.514166:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2378:1862022098731392:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764883.514177:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098731648:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764884.586753:0:66981:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf800 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66981:1862022098732032:192.168.202.128@tcp:60:lhsmtool_posix.0 00000080:00200008:1.0:1775764884.586839:0:66981:0:(file.c:447:ll_file_release()) START file fid:[0x200000002:0x2:0x0](ffff8b2390ec5348), flags 100000 00000002:00100000:1.0:1775764884.586899:0:66981:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864f0000 x1862022098698624/t34359739360(34359739360) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/4688 e 0 to 0 dl 1775764869 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764884.586912:0:66981:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764884.586916:0:66981:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd500 x1862022098732288/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764884.586940:0:66981:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66981:1862022098732288:192.168.202.128@tcp:35:lhsmtool_posix.0 00000100:00100000:1.0:1775764884.587238:0:66981:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238495b480 going to sleep for 11 seconds 00000100:00100000:1.0:1775764884.605956:0:66981:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job lhsmtool_posix:fcf15866-4739-4595-8fe1-44abde7ee9d7:66981:1862022098732288:192.168.202.128@tcp:35:lhsmtool_posix.0 00000002:00100000:1.0:1775764884.605998:0:66981:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864f0000 x1862022098698624/t34359739360(34359739360) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/4688 e 0 to 0 dl 1775764869 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1775764884.606074:0:66981:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 100000 00000080:00200008:2.0:1775764884.613848:0:66981:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 100000 00010000:00010000:1.0:1775764886.456906:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764888.507003:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764890.552278:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1775764890.617002:0:68216:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1775764890.617015:0:68216:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764890.617037:0:68216:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764890.617055:0:68216:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764890.617084:0:68216:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:2.0:1775764890.617093:0:68216:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764890.617102:0:68216:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764890.617105:0:68216:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c388 00000080:00010000:2.0:1775764890.617113:0:68216:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c388 from it ffffa60b01e03cb8 00010000:00010000:2.0:1775764890.617116:0:68216:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1775764890.617124:0:68216:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200008:2.0:1775764890.618541:0:68216:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b2389909148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1775764890.618548:0:68216:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:2.0:1775764890.618563:0:68216:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764890.618575:0:68216:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17c396 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030532ae expref: -99 pid: 67940 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764890.618588:0:68216:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17c396 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030532ae expref: -99 pid: 67940 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764890.618595:0:68216:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764890.618601:0:68216:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:2.0:1775764890.618604:0:68216:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17c396 00000080:00010000:2.0:1775764890.618608:0:68216:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c396 from it ffffa60b01e03cb8 00010000:00010000:2.0:1775764890.618611:0:68216:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17c396 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030532ae expref: -99 pid: 67940 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764890.618618:0:68216:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17c396 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030532ae expref: -99 pid: 67940 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775764890.618824:0:68216:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764890.618834:0:68216:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764890.618843:0:68216:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764890.618853:0:68216:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:2.0:1775764890.618864:0:68216:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764890.618868:0:68216:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764890.618870:0:68216:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c388 00000080:00010000:2.0:1775764890.618874:0:68216:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c388 from it ffffa60b01e03bb0 00010000:00010000:2.0:1775764890.618877:0:68216:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1775764890.618884:0:68216:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:2.0:1775764890.618894:0:68216:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764890.618901:0:68216:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 100000 00000080:00200000:2.0:1775764890.737204:0:68225:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764890.737245:0:68225:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764890.737269:0:68225:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1775764890.737290:0:68225:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1775764890.737299:0:68225:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764890.737310:0:68225:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764890.737314:0:68225:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c388 00000080:00010000:2.0:1775764890.737324:0:68225:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c388 from it ffffa60b01e03bb0 00010000:00010000:2.0:1775764890.737328:0:68225:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1775764890.737344:0:68225:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:2.0:1775764890.737363:0:68225:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:2.0:1775764890.737371:0:68225:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200080:2.0:1775764890.737402:0:68225:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=8080667f arg=7ffee161ad60 00000080:00200008:2.0:1775764890.737420:0:68225:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 504000 00000080:00200000:1.0:1775764890.756861:0:67986:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764890.756898:0:67986:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764890.756924:0:67986:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764890.756944:0:67986:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:1.0:1775764890.756954:0:67986:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764890.756964:0:67986:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764890.756969:0:67986:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c388 00000080:00010000:1.0:1775764890.756978:0:67986:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c388 from it ffffa60b01d7bbf8 00010000:00010000:1.0:1775764890.756982:0:67986:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1775764890.756992:0:67986:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:1.0:1775764890.757012:0:67986:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764890.757030:0:67986:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764890.757035:0:67986:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764890.757042:0:67986:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:1.0:1775764890.757050:0:67986:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:1.0:1775764890.757072:0:67986:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764890.757076:0:67986:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764890.757079:0:67986:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c388 00000080:00010000:1.0:1775764890.757082:0:67986:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c388 from it ffffa60b01d7bb38 00010000:00010000:1.0:1775764890.757085:0:67986:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:1.0:1775764890.757093:0:67986:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:1.0:1775764890.757101:0:67986:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 3 00000080:00200000:1.0:1775764890.757106:0:67986:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f28.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), file ffff8b2385294900, open_flags 8241, mode 81b6 opened 0 00000080:00200000:1.0:1775764890.757120:0:67986:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f28.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000002:00010000:1.0:1775764890.757146:0:67986:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f28.sanity-pcc,[0x200003ab4:0x24:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0101102 00010000:00010000:1.0:1775764890.757159:0:67986:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1775764890.757225:0:67986:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1775764890.757232:0:67986:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764890.757234:0:67986:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031e00/0x73cfd86cfc17c388 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03053268 expref: -99 pid: 67469 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1775764890.757275:0:67986:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67986 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.757287:0:67986:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67986 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764890.757300:0:67986:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764890.757316:0:67986:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fd500 x1862022098733696/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764890.757327:0:67986:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67986 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764890.757354:0:67986:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:67986:1862022098733696:192.168.202.128@tcp:101:bash.0 00000100:00100000:1.0:1775764890.757579:0:67986:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654b40 going to sleep for 11 seconds 00000100:00100000:1.0:1775764890.769658:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119092352 00000100:00100000:1.0:1775764890.769673:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119092352 00000100:00100000:1.0:1775764890.770954:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 318 00000100:00100000:1.0:1775764890.770975:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daf800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42325:x1862022119092352:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764890.771001:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b9c5fa00/0x73cfd86cfc17c396 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x57b8ff0e030532ae expref: -99 pid: 67940 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764890.771132:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764890.771139:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daf800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42325:x1862022119092352:12345-192.168.202.128@tcp:104: Request processed in 174us (1792us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764890.771148:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 318 00000100:00100000:1.0:1775764890.789427:0:67986:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:67986:1862022098733696:192.168.202.128@tcp:101:bash.0 00010000:00010000:1.0:1775764890.789506:0:67986:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.789557:0:67986:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.789567:0:67986:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.789580:0:67986:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.789593:0:67986:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.789608:0:67986:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.789617:0:67986:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.789624:0:67986:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764890.789638:0:67986:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8b23864fd500 x1862022098733696/t34359739451(34359739451) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764906 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764890.789653:0:67986:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fd500 x1862022098733696/t34359739451(34359739451) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764906 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764890.789671:0:67986:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764890.789687:0:67986:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.789699:0:67986:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/36 (0 0) 00000080:00200000:1.0:1775764890.789709:0:67986:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1775764890.789729:0:67986:0:(namei.c:1620:ll_create_it()) VFS Op:name=f28.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=open|creat 00000080:00200000:1.0:1775764890.789784:0:67986:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab4:0x24:0x0](ffff8b2389a642c8) 00000080:00200000:1.0:1775764890.789961:0:67986:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x24:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:1.0:1775764890.789973:0:67986:0:(namei.c:1586:ll_create_node()) setting l_ast_data to inode [0x200003ab4:0x24:0x0](ffff8b2389a642c8) 00000080:00010000:1.0:1775764890.789977:0:67986:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x24:0x0](ffff8b2389a642c8) for lock 0x73cfd86cfc17c3ab 00000080:00010000:1.0:1775764890.789990:0:67986:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x24:0x0](ffff8b2389a642c8) 00000080:00200008:1.0:1775764890.790010:0:67986:0:(file.c:1057:ll_file_open()) START file f28.sanity-pcc:[0x200003ab4:0x24:0x0](ffff8b2389a642c8), flags 101101 00000002:00100000:1.0:1775764890.790033:0:67986:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fd500 x1862022098733696/t34359739451(34359739451) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764906 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1775764890.790096:0:67986:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3ab from it ffff8b2389ed61e0 00010000:00010000:1.0:1775764890.790102:0:67986:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.790112:0:67986:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764890.790155:0:67986:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x24:0x0](ffff8b2389a642c8), inode mode 81a4 mask 40 00000080:00200008:1.0:1775764890.790257:0:67986:0:(file.c:2612:do_file_write_iter()) START file f28.sanity-pcc:[0x200003ab4:0x24:0x0], ppos: 0, count: 22 00000080:00200000:1.0:1775764890.790272:0:67986:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 22 bytes 00000080:00200000:1.0:1775764890.790280:0:67986:0:(file.c:2588:ll_do_tiny_write()) result: 0, original count 22 00000080:00200000:1.0:1775764890.790295:0:67986:0:(file.c:2058:ll_file_io_generic()) f28.sanity-pcc: write ppos: 0, bytes: 22 00000080:00200000:1.0:1775764890.790301:0:67986:0:(file.c:1815:ll_io_set_mirror()) f28.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764890.790309:0:67986:0:(cl_io.c:214:cl_io_rw_init()) header@0000000023f5efcf[0x110, 1, [0x200003ab4:0x24:0x0] hash] 00000020:00200000:1.0:1775764890.790312:0:67986:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 22) 0 0 00000080:00200000:1.0:1775764890.790317:0:67986:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x24:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764890.790337:0:67986:0:(file.c:2136:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:1.0:1775764890.790347:0:67986:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 22) 22, 22 00020000:00200000:1.0:1775764890.790351:0:67986:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764890.790383:0:67986:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:1.0:1775764890.790399:0:67986:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764890.790406:0:67986:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c54b0, osc lock ffff8b2382cf6980, flags 0x0 00010000:00010000:1.0:1775764890.790515:0:67986:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.790525:0:67986:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.790538:0:67986:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764890.790561:0:67986:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:67986:1862022098733952:192.168.202.128@tcp:101:bash.0 00000100:00100000:1.0:1775764890.790669:0:67986:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764890.794358:0:67986:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:67986:1862022098733952:192.168.202.128@tcp:101:bash.0 00010000:00010000:1.0:1775764890.794397:0:67986:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.794422:0:67986:0:(ldlm_resource.c:1714:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.794438:0:67986:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.794679:0:67986:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1775764890.794696:0:67986:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.794709:0:67986:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.794718:0:67986:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1775764890.794753:0:67986:0:(vvp_io.c:1286:vvp_io_write_start()) f28.sanity-pcc: write [0, 22) append 0 append lockpos 0 00000080:00200000:1.0:1775764890.794778:0:67986:0:(rw26.c:663:ll_write_begin()) Writing 0 of 0 to 22 bytes 00000080:00200000:1.0:1775764890.794950:0:67986:0:(rw26.c:858:ll_write_end()) pos 0, len 22, copied 22 00000080:00200000:1.0:1775764890.794964:0:67986:0:(rw26.c:916:ll_write_end()) page@00000000a4122f7e[3 000000003da9ebb0 2 1 000000002f47ea13] 00000080:00200000:1.0:1775764890.794968:0:67986:0:(rw26.c:916:ll_write_end()) vmpage @0000000001bcb392 17ffffc0001001 3:0 ffff8b238a05f8c0 0 lru 00000080:00200000:1.0:1775764890.794976:0:67986:0:(rw26.c:916:ll_write_end()) osc-page@00000000f2c35a15 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 000000003596ffcd 00000000a22dd662 > 3< 0 0 > 4< 0 0 8 22151168 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1775764890.794979:0:67986:0:(rw26.c:916:ll_write_end()) end page@00000000a4122f7e 00000080:00200000:1.0:1775764890.794981:0:67986:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764890.794986:0:67986:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 22 prio 0 00000008:00010000:1.0:1775764890.795010:0:67986:0:(osc_cache.c:848:osc_extent_find()) ### extent: 0000000022e0e21a ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1775764890.795032:0:67986:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff8b2389a64478, count 1, dirtied 1 00000080:00200000:1.0:1775764890.795037:0:67986:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 22 bytes, tot: 22 00000080:00200000:1.0:1775764890.795040:0:67986:0:(vvp_io.c:1353:vvp_io_write_start()) f28.sanity-pcc: write bytes 0, result: 22 00000020:00010000:1.0:1775764890.795071:0:67986:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb73f8 (ffff8b2387cba2d8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764890.795075:0:67986:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb73f8 (ffff8b2387cba2d8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764890.795079:0:67986:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c54b0 (ffff8b2387cbaf38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764890.795085:0:67986:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764890.795095:0:67986:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:1.0:1775764890.795115:0:67986:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb73f8 (ffff8b2387cba2d8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764890.795119:0:67986:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c54b0 (ffff8b2387cba2d8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764890.795129:0:67986:0:(vvp_io.c:517:vvp_io_advance()) advancing 22 bytes 00000080:00200000:1.0:1775764890.795139:0:67986:0:(file.c:2173:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:1.0:1775764890.795177:0:67986:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x24:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764890.795181:0:67986:0:(file.c:2206:ll_file_io_generic()) f28.sanity-pcc: 2 io complete with rc: 0, result: 22, restart: 0 00000080:00200000:1.0:1775764890.795190:0:67986:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x24:0x0] updating i_size 22 i_blocks 1 00000080:00200000:1.0:1775764890.795195:0:67986:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 22 00000080:00200008:1.0:1775764890.795257:0:67986:0:(file.c:447:ll_file_release()) START file f28.sanity-pcc:[0x200003ab4:0x24:0x0](ffff8b2389a642c8), flags 100001 00000002:00100000:1.0:1775764890.795330:0:67986:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fd500 x1862022098733696/t34359739451(34359739451) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764906 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764890.795344:0:67986:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764890.795349:0:67986:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ffb80 x1862022098734208/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764890.795375:0:67986:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:67986:1862022098734208:192.168.202.128@tcp:35:bash.0 00000100:00100000:1.0:1775764890.795503:0:67986:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764890.801251:0:67986:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job bash:fcf15866-4739-4595-8fe1-44abde7ee9d7:67986:1862022098734208:192.168.202.128@tcp:35:bash.0 00000002:00100000:1.0:1775764890.801299:0:67986:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fd500 x1862022098733696/t34359739451(34359739451) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764906 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1775764891.258011:0:68241:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764891.258054:0:68241:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764891.258980:0:68241:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3b9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68241 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.258996:0:68241:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3b9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68241 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.259011:0:68241:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3b9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68241 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764891.259048:0:68241:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:68241:1862022098734336:192.168.202.128@tcp:101:multiop.0 00000100:00100000:1.0:1775764891.259243:0:68241:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764891.266001:0:68241:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:68241:1862022098734336:192.168.202.128@tcp:101:multiop.0 00010000:00010000:1.0:1775764891.266038:0:68241:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3b9 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053325 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.266053:0:68241:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3b9 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053325 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.266079:0:68241:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3b9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053325 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.266091:0:68241:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3b9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053325 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.266111:0:68241:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3b9 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053325 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.266120:0:68241:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3b9 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053325 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.266127:0:68241:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3b9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053325 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764891.266139:0:68241:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864fea00 x1862022098734336/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764907 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764891.266154:0:68241:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053325 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.266170:0:68241:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1775764891.266178:0:68241:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764891.266216:0:68241:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:1.0:1775764891.266223:0:68241:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c3b9 00000080:00010000:1.0:1775764891.266230:0:68241:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764891.266233:0:68241:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3b9 from it ffffa60b01e6bbf8 00010000:00010000:1.0:1775764891.266237:0:68241:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053325 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.266246:0:68241:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053325 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764891.266282:0:68241:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764891.266304:0:68241:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f28.sanity-pcc, flags=777 00000080:00200000:1.0:1775764891.266316:0:68241:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x24:0x0](ffff8b2389a642c8), inode mode 81a4 mask 46 00000080:00200008:1.0:1775764891.266328:0:68241:0:(file.c:1057:ll_file_open()) START file f28.sanity-pcc:[0x200003ab4:0x24:0x0](ffff8b2389a642c8), flags 100102 00000002:00010000:1.0:1775764891.266356:0:68241:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x24:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0240100103 00010000:00010000:1.0:1775764891.266372:0:68241:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03053325 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764891.266440:0:68241:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3b9 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03053325 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764891.266448:0:68241:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764891.266503:0:68241:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3b9 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03053325 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764891.266546:0:68241:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68241 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.266554:0:68241:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68241 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764891.266567:0:68241:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764891.266574:0:68241:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fea00 x1862022098734592/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764891.266585:0:68241:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68241 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764891.266620:0:68241:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:68241:1862022098734592:192.168.202.128@tcp:101:multiop.0 00000100:00100000:1.0:1775764891.266806:0:68241:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1775764891.274141:0:68241:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:68241:1862022098734592:192.168.202.128@tcp:101:multiop.0 00010000:00010000:1.0:1775764891.274180:0:68241:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053333 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.274219:0:68241:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053333 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.274229:0:68241:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053333 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.274244:0:68241:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053333 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.274259:0:68241:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053333 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.274276:0:68241:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053333 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.274286:0:68241:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053333 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.274295:0:68241:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053333 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764891.274308:0:68241:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=2b, status=0 req@ffff8b23864fea00 x1862022098734592/t34359739453(34359739453) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764907 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764891.274324:0:68241:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fea00 x1862022098734592/t34359739453(34359739453) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764907 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764891.274340:0:68241:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053333 expref: -99 pid: 68241 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764891.274358:0:68241:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053333 expref: -99 pid: 68241 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.274440:0:68241:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764891.274579:0:68241:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764891.274588:0:68241:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053333 expref: -99 pid: 68241 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.274595:0:68241:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e03053333 expref: -99 pid: 68241 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.274684:0:68241:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e03053333 expref: -99 pid: 68241 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764891.274765:0:68241:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:1.0:1775764891.274775:0:68241:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764891.274911:0:68241:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x24:0x0]: layout version change: 0 -> 0 00000080:00010000:1.0:1775764891.274923:0:68241:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x24:0x0](ffff8b2389a642c8) for lock 0x73cfd86cfc17c3ab 00000080:00010000:1.0:1775764891.274932:0:68241:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3ab from it ffffa60b01e6bbf0 00010000:00010000:1.0:1775764891.274936:0:68241:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764891.274945:0:68241:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000002:00100000:1.0:1775764891.274977:0:68241:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864fea00 x1862022098734592/t34359739453(34359739453) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764907 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1775764891.277504:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775764891.277583:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03053333 expref: -99 pid: 68241 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764891.277592:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764891.277625:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03053333 expref: -99 pid: 68241 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764891.277634:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c0 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03053333 expref: -99 pid: 68241 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764891.277658:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764891.277660:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764891.277662:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764891.277674:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764891.277702:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098734720:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764891.283177:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098734720:192.168.202.128@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764892.599669:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764893.752530:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2379:1862022098734848:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764893.753299:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098735104:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764893.753338:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098735360:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764893.767887:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098735360:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764893.768086:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:24b54882-c0a1-43f2-954f-009e5a9afc9d:2379:1862022098734848:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764893.768102:0:2379:0:(client.c:3082:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b2389398000: committing for last_committed 34359739453 gen 1 00000100:00100000:2.0:1775764893.768113:0:2379:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23864f2300 x1862022098733312/t34359739450(34359739450) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/4688 e 0 to 0 dl 1775764906 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lhsmtool_posix.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764893.768138:0:2379:0:(client.c:3131:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b23864fea00 x1862022098734592/t34359739453(34359739453) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764907 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1775764893.768146:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098735104:192.168.202.128@tcp:400:kworker.0 00000080:00200008:1.0:1775764894.135824:0:68269:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764894.135841:0:68269:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764894.135865:0:68269:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764894.135936:0:68269:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68269 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.135947:0:68269:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68269 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.135957:0:68269:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68269 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764894.135985:0:68269:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:68269:1862022098735488:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764894.136224:0:68269:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654480 going to sleep for 11 seconds 00000100:00100000:1.0:1775764894.142076:0:68269:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:68269:1862022098735488:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764894.142107:0:68269:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.142119:0:68269:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.142133:0:68269:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.142146:0:68269:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.142159:0:68269:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.142167:0:68269:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.142174:0:68269:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764894.142185:0:68269:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864ff800 x1862022098735488/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764910 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764894.142200:0:68269:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.142213:0:68269:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1775764894.142220:0:68269:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1775764894.142251:0:68269:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:1.0:1775764894.142257:0:68269:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c3c7 00000080:00010000:1.0:1775764894.142262:0:68269:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764894.142266:0:68269:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3c7 from it ffffa60b01e7bcb8 00010000:00010000:1.0:1775764894.142269:0:68269:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.142278:0:68269:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764894.142355:0:68269:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764894.142372:0:68269:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764894.142382:0:68269:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764894.142393:0:68269:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1775764894.142400:0:68269:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764894.142404:0:68269:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764894.142407:0:68269:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c3c7 00000080:00010000:1.0:1775764894.142411:0:68269:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3c7 from it ffffa60b01e7bb58 00010000:00010000:1.0:1775764894.142414:0:68269:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764894.142421:0:68269:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764894.142429:0:68269:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764894.142437:0:68269:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f28.sanity-pcc, flags=8 00000080:00200008:1.0:1775764894.142446:0:68269:0:(file.c:6226:ll_getattr_dentry()) START file f28.sanity-pcc:[0x200003ab4:0x24:0x0](ffff8b2389a642c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1775764894.142450:0:68269:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x24:0x0](ffff8b2389a642c8),name=f28.sanity-pcc 00000002:00010000:1.0:1775764894.142484:0:68269:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x24:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764894.142493:0:68269:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764894.142501:0:68269:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764894.142508:0:68269:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764894.142512:0:68269:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x24:0x0](ffff8b2389a642c8) 00000080:00010000:1.0:1775764894.142514:0:68269:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x24:0x0](ffff8b2389a642c8) for lock 0x73cfd86cfc17c3ab 00000080:00010000:1.0:1775764894.142518:0:68269:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3ab from it ffffa60b01e7bcb8 00010000:00010000:1.0:1775764894.142522:0:68269:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764894.142529:0:68269:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764894.142609:0:68269:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x24:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1775764894.142618:0:68269:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x24:0x0] 00000008:00010000:1.0:1775764894.142649:0:68269:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764894.142652:0:68269:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5330, osc lock ffff8b2382cf6980, flags 0x400001002 00010000:00010000:1.0:1775764894.142669:0:68269:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764894.142681:0:68269:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1775764894.142693:0:68269:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764894.142696:0:68269:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764894.142704:0:68269:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 5/0,0 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1775764894.142713:0:68269:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764894.142723:0:68269:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x24:0x0] updating i_size 22 i_blocks 1 00000020:00010000:1.0:1775764894.142726:0:68269:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1b6c0 (ffff8b2387cba2d8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764894.142730:0:68269:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1b6c0 (ffff8b2387cba2d8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764894.142732:0:68269:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5330 (ffff8b2387cbaf38/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764894.142736:0:68269:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1b6c0 (ffff8b2387cba2d8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764894.142738:0:68269:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5330 (ffff8b2387cba2d8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764894.142756:0:68269:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x24:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764894.142792:0:68269:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764894.142800:0:68269:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764894.142807:0:68269:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764894.142815:0:68269:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764894.142821:0:68269:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764894.142824:0:68269:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764894.142827:0:68269:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c3c7 00000080:00010000:1.0:1775764894.142830:0:68269:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3c7 from it ffffa60b01e7bbf8 00010000:00010000:1.0:1775764894.142833:0:68269:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764894.142846:0:68269:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764894.142854:0:68269:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1775764894.142859:0:68269:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f28.sanity-pcc, flags=265 00000080:00200000:1.0:1775764894.142864:0:68269:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x24:0x0](ffff8b2389a642c8), inode mode 81a4 mask 46 00000080:00200008:1.0:1775764894.142871:0:68269:0:(file.c:1057:ll_file_open()) START file f28.sanity-pcc:[0x200003ab4:0x24:0x0](ffff8b2389a642c8), flags 104002 00000080:00200080:1.0:1775764894.142945:0:68269:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x24:0x0](ffff8b2389a642c8) cmd=c00c66f3 arg=7fffd69eb984 00000080:00200008:1.0:1775764894.143106:0:68269:0:(file.c:447:ll_file_release()) START file f28.sanity-pcc:[0x200003ab4:0x24:0x0](ffff8b2389a642c8), flags 104002 00000080:00200008:2.0:1775764894.351737:0:68241:0:(file.c:447:ll_file_release()) START file f28.sanity-pcc:[0x200003ab4:0x24:0x0](ffff8b2389a642c8), flags 100002 00000002:00100000:2.0:1775764894.351803:0:68241:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864fea00 x1862022098734592/t34359739453(34359739453) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764907 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764894.351817:0:68241:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764894.351822:0:68241:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4700 x1862022098735744/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764894.351860:0:68241:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:68241:1862022098735744:192.168.202.128@tcp:35:multiop.0 00000100:00100000:2.0:1775764894.351949:0:68241:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85d4a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764894.367190:0:68241:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job multiop:fcf15866-4739-4595-8fe1-44abde7ee9d7:68241:1862022098735744:192.168.202.128@tcp:35:multiop.0 00000002:00100000:2.0:1775764894.367224:0:68241:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23864fea00 x1862022098734592/t34359739453(34359739453) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764907 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764894.480418:0:68290:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:68290:1862022098736000:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764894.480439:0:68290:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.480493:0:68290:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.480503:0:68290:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.480515:0:68290:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.480526:0:68290:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.480539:0:68290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.480547:0:68290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.480554:0:68290:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764894.480564:0:68290:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b2382dad880 x1862022098736000/t34359739455(34359739455) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764910 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764894.480577:0:68290:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382dad880 x1862022098736000/t34359739455(34359739455) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764910 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764894.480588:0:68290:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1775764894.480601:0:68290:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.480612:0:68290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764894.480623:0:68290:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764894.480631:0:68290:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.480638:0:68290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.480646:0:68290:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.480665:0:68290:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:1.0:1775764894.480672:0:68290:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00010000:00010000:2.0:1775764894.480702:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764894.480710:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.480717:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2381497600 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764894.480721:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.480729:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.480736:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.480751:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1775764894.480774:0:68290:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x24:0x0]: layout version change: 0 -> 0 00000080:00010000:1.0:1775764894.480783:0:68290:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x24:0x0](ffff8b2389a642c8) for lock 0x73cfd86cfc17c3ab 00010000:00010000:2.0:1775764894.480784:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000080:00010000:1.0:1775764894.480789:0:68290:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3ab from it ffffa60b01f7fbf0 00010000:00010000:2.0:1775764894.480790:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1775764894.480792:0:68290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1775764894.480799:0:68290:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764894.480813:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.480821:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c3ce lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e03053356 expref: -99 pid: 68290 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.480836:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764894.480837:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764894.480839:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764894.480847:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764894.480861:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098736128:192.168.202.128@tcp:103:ldlm_bl.0 00000002:00100000:1.0:1775764894.481006:0:68290:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b2382dad880 x1862022098736000/t34359739455(34359739455) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764910 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1775764894.481134:0:68290:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x24:0x0](ffff8b2389a642c8) cmd=c00c66f3 arg=7fff75e244e4 00000002:00010000:1.0:1775764894.481152:0:68290:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x24:0x0]) in obj [0x200003ab4:0x24:0x0], intent: open flags 01004040000002 00010000:00010000:1.0:1775764894.481187:0:68290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c3d5 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x24:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68290 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.481194:0:68290:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x81000000001000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c3d5 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68290 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1775764894.481204:0:68290:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1775764894.481209:0:68290:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864ff800 x1862022098736384/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1775764894.481216:0:68290:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c3d5 lrc: 3/1,0 mode: --/CR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x81000000000000 pid: 68290 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764894.481227:0:68290:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:68290:1862022098736384:192.168.202.128@tcp:101:lfs.0 00000100:00100000:1.0:1775764894.481786:0:68290:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654c00 going to sleep for 11 seconds 00000100:00100000:2.0:1775764894.490598:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098736128:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764894.490642:0:68290:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:68290:1862022098736384:192.168.202.128@tcp:101:lfs.0 00010000:00010000:1.0:1775764894.490662:0:68290:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PW ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c3d5 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e0305336b expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.490674:0:68290:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c3d5 lrc: 4/1,0 mode: --/PW res: [0x200003ab4:0x24:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e0305336b expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.490685:0:68290:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c3d5 lrc: 4/1,0 mode: PW/PW res: [0x200003ab4:0x24:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e0305336b expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.490697:0:68290:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c3d5 lrc: 4/1,0 mode: PW/PW res: [0x200003ab4:0x24:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e0305336b expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.490720:0:68290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c3d5 lrc: 5/1,1 mode: PW/PW res: [0x200003ab4:0x24:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e0305336b expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.490727:0:68290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c3d5 lrc: 5/1,1 mode: PW/PW res: [0x200003ab4:0x24:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e0305336b expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.490734:0:68290:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c3d5 lrc: 4/0,1 mode: PW/PW res: [0x200003ab4:0x24:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e0305336b expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764894.490750:0:68290:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=600002b, status=0 req@ffff8b23864ff800 x1862022098736384/t34359739456(34359739456) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764910 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1775764894.490763:0:68290:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864ff800 x1862022098736384/t34359739456(34359739456) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764910 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1775764894.490776:0:68290:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c3d5 lrc: 3/0,1 mode: PW/PW res: [0x200003ab4:0x24:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x57b8ff0e0305336b expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.490790:0:68290:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 2 res 8589949620/36 (0 0) 00000080:00200000:1.0:1775764894.490796:0:68290:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200003ab4:0x24:0x0] suppgids 0 0: rc 0 00000002:00100000:1.0:1775764894.490823:0:68290:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23864ff800 x1862022098736384/t34359739456(34359739456) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764910 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1775764894.490832:0:68290:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x24:0x0](ffff8b2389a642c8) for lock 0x73cfd86cfc17c3d5 00000080:00010000:1.0:1775764894.490837:0:68290:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3d5 from it ffffa60b01f7fcc0 00010000:00010000:1.0:1775764894.490840:0:68290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c3d5 lrc: 3/0,1 mode: PW/PW res: [0x200003ab4:0x24:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e0305336b expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.490844:0:68290:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c3d5 lrc: 2/0,0 mode: PW/PW res: [0x200003ab4:0x24:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x57b8ff0e0305336b expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200080:1.0:1775764894.491166:0:68290:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x24:0x0](ffff8b2389a642c8) cmd=c00c66f3 arg=15a92a0 00000080:00200008:1.0:1775764894.492252:0:68290:0:(file.c:2454:do_file_read_iter()) START file f28.sanity-pcc:[0x200003ab4:0x24:0x0], ppos: 0, count: 1048576 00000080:00200000:1.0:1775764894.492306:0:68290:0:(file.c:2058:ll_file_io_generic()) f28.sanity-pcc: read ppos: 22, bytes: 1048554 00000080:00200000:1.0:1775764894.492311:0:68290:0:(file.c:1815:ll_io_set_mirror()) f28.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764894.492319:0:68290:0:(cl_io.c:214:cl_io_rw_init()) header@0000000023f5efcf[0x110, 2, [0x200003ab4:0x24:0x0] hash] 00000020:00200000:1.0:1775764894.492322:0:68290:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [22, 1048576) 2048 0 00000080:00200000:1.0:1775764894.492328:0:68290:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x24:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764894.492347:0:68290:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 22 chunk: [22, 1048576) 1048576, 1048554 00020000:00200000:1.0:1775764894.492351:0:68290:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764894.493546:0:68290:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:1.0:1775764894.493569:0:68290:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 1 00000008:00010000:1.0:1775764894.493574:0:68290:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5330, osc lock ffff8b2382cf6980, flags 0x40000 00010000:00010000:1.0:1775764894.493600:0:68290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764894.493615:0:68290:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764894.493627:0:68290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764894.493637:0:68290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764894.493643:0:68290:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1775764894.493649:0:68290:0:(vvp_io.c:826:vvp_io_read_start()) f28.sanity-pcc: read [22, 1048576) 00000001:00010000:1.0:1775764894.493661:0:68290:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x24:0x0] 00000008:00010000:1.0:1775764894.493666:0:68290:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764894.493667:0:68290:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c59b0, osc lock ffff8b2382cf7a20, flags 0x400001002 00010000:00010000:1.0:1775764894.493672:0:68290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764894.493678:0:68290:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1775764894.493685:0:68290:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764894.493688:0:68290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 8/1,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764894.493695:0:68290:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 7/1,0 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1775764894.493700:0:68290:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764894.493873:0:68290:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x24:0x0] updating i_size 22 i_blocks 1 00000020:00010000:1.0:1775764894.493876:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1b6c0 (ffff8b2387cba2d8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764894.493878:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1b6c0 (ffff8b2387cba2d8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764894.493880:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c59b0 (ffff8b2387cbaf38/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764894.493884:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1b6c0 (ffff8b2387cba2d8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764894.493885:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c59b0 (ffff8b2387cba2d8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764894.493893:0:68290:0:(vvp_io.c:885:vvp_io_read_start()) count:1048554 iocb pos:22 00000020:00010000:1.0:1775764894.493902:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb73f8 (ffff8b2387cba2d8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764894.493903:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb73f8 (ffff8b2387cba2d8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764894.493905:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5330 (ffff8b2387cbaf38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764894.493907:0:68290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1775764894.493967:0:68290:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:1.0:1775764894.493980:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb73f8 (ffff8b2387cba2d8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764894.493981:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5330 (ffff8b2387cba2d8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764894.493987:0:68290:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:1.0:1775764894.494015:0:68290:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x24:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764894.494019:0:68290:0:(file.c:2206:ll_file_io_generic()) f28.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:1.0:1775764894.494022:0:68290:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:1.0:1775764894.494134:0:68290:0:(file.c:2454:do_file_read_iter()) START file f28.sanity-pcc:[0x200003ab4:0x24:0x0], ppos: 22, count: 1048576 00000080:00200000:1.0:1775764894.494151:0:68290:0:(file.c:2058:ll_file_io_generic()) f28.sanity-pcc: read ppos: 22, bytes: 1048576 00000080:00200000:1.0:1775764894.494154:0:68290:0:(file.c:1815:ll_io_set_mirror()) f28.sanity-pcc: desiginated mirror: 0 00000020:00200000:1.0:1775764894.494157:0:68290:0:(cl_io.c:214:cl_io_rw_init()) header@0000000023f5efcf[0x110, 2, [0x200003ab4:0x24:0x0] hash] 00000020:00200000:1.0:1775764894.494159:0:68290:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [22, 1048598) 2048 0 00000080:00200000:1.0:1775764894.494161:0:68290:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x24:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764894.494171:0:68290:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 22 chunk: [22, 1048598) 1048598, 1048576 00020000:00200000:1.0:1775764894.494173:0:68290:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764894.494185:0:68290:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 0 [0, 256] 00000008:00010000:1.0:1775764894.494189:0:68290:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 1 00000008:00010000:1.0:1775764894.494191:0:68290:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c5330, osc lock ffff8b2382cf6980, flags 0x40000 00010000:00010000:1.0:1775764894.494198:0:68290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764894.494204:0:68290:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1052671) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764894.494210:0:68290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764894.494214:0:68290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764894.494218:0:68290:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1775764894.494226:0:68290:0:(vvp_io.c:826:vvp_io_read_start()) f28.sanity-pcc: read [22, 1048598) 00000001:00010000:1.0:1775764894.494232:0:68290:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200003ab4:0x24:0x0] 00000008:00010000:1.0:1775764894.494234:0:68290:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1775764894.494235:0:68290:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff8b23a79c59b0, osc lock ffff8b2382cf7a20, flags 0x400001002 00010000:00010000:1.0:1775764894.494238:0:68290:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764894.494243:0:68290:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1775764894.494246:0:68290:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1775764894.494248:0:68290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 8/1,1 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764894.494252:0:68290:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 7/1,0 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1775764894.494256:0:68290:0:(cl_io.c:1822:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1775764894.494258:0:68290:0:(file.c:1742:ll_merge_attr_nolock()) [0x200003ab4:0x24:0x0] updating i_size 22 i_blocks 1 00000020:00010000:1.0:1775764894.494259:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2384f1b6c0 (ffff8b2387cba2d8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764894.494261:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2384f1b6c0 (ffff8b2387cba2d8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764894.494262:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c59b0 (ffff8b2387cbaf38/1) at cl_lock_cancel():142 00000020:00010000:1.0:1775764894.494264:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2384f1b6c0 (ffff8b2387cba2d8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764894.494265:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c59b0 (ffff8b2387cba2d8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764894.494268:0:68290:0:(vvp_io.c:885:vvp_io_read_start()) count:1048576 iocb pos:22 00000020:00010000:1.0:1775764894.494271:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b2387cb73f8 (ffff8b2387cba2d8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764894.494273:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b2387cb73f8 (ffff8b2387cba2d8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764894.494274:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b23a79c5330 (ffff8b2387cbaf38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764894.494276:0:68290:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1775764894.494281:0:68290:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00010000:1.0:1775764894.494286:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b2387cb73f8 (ffff8b2387cba2d8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764894.494287:0:68290:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b23a79c5330 (ffff8b2387cba2d8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764894.494289:0:68290:0:(vvp_io.c:517:vvp_io_advance()) advancing 0 bytes 00000080:00200000:1.0:1775764894.494295:0:68290:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x24:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764894.494297:0:68290:0:(file.c:2206:ll_file_io_generic()) f28.sanity-pcc: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:1.0:1775764894.494299:0:68290:0:(file.c:2292:ll_file_io_generic()) iot: 1, result: 0 00000080:00200000:1.0:1775764894.494437:0:68290:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x24:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1775764894.494441:0:68290:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:1.0:1775764894.500968:0:2368:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000008:00010000:2.0:1775764894.512944:0:2378:0:(osc_cache.c:901:osc_extent_finish()) ### extent: 0000000022e0e21a ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800429400000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000008:00010000:2.0:1775764894.513792:0:2378:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000022e0e21a ns: lustre-OST0000-osc-ffff8b2389398000 lock: 000000006ae897c9/0x73cfd86cfc17c3b2 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x120:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800429400000000 nid: local remote: 0x57b8ff0e03053317 expref: -99 pid: 67986 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:2.0:1775764894.513831:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098736640:192.168.202.128@tcp:4:lfs.0 00000100:00100000:2.0:1775764894.514349:0:2378:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098736768:192.168.202.128@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1775764894.534020:0:68290:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c3d5 lrc: 2/0,0 mode: PW/PW res: [0x200003ab4:0x24:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x57b8ff0e0305336b expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.534054:0:68290:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741a00/0x73cfd86cfc17c3d5 lrc: 0/0,0 mode: --/PW res: [0x200003ab4:0x24:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x57b8ff0e0305336b expref: -99 pid: 68290 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1775764894.534076:0:68290:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23864ff800 x1862022098736384/t34359739456(34359739456) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764910 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1775764894.534088:0:68290:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1775764894.534098:0:68290:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23864fce00 x1862022098737024/t0(0) o35->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:23/10 lens 504/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1775764894.534128:0:68290:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:68290:1862022098737024:192.168.202.128@tcp:35:lfs.0 00000100:00100000:1.0:1775764894.534232:0:68290:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654c00 going to sleep for 11 seconds 00000100:00100000:1.0:1775764894.544350:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764894.544866:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764894.544874:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764894.544883:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b23b8741000 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764894.544888:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764894.544897:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764894.544910:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1775764894.544960:0:46149:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x24:0x0]: losing layout lock 00010000:00010000:2.0:1775764894.545282:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764894.545291:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764894.545317:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764894.545323:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8741000/0x73cfd86cfc17c3ab lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e03053309 expref: -99 pid: 67986 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1775764894.545344:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764894.545347:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764894.545348:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1775764894.795321:0:68306:0:(file.c:6226:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764894.795337:0:68306:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764894.795358:0:68306:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764894.795378:0:68306:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764894.795395:0:68306:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1775764894.795404:0:68306:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764894.795413:0:68306:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764894.795416:0:68306:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c3c7 00000080:00010000:2.0:1775764894.795423:0:68306:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3c7 from it ffffa60b01fafcb8 00010000:00010000:2.0:1775764894.795427:0:68306:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1775764894.795436:0:68306:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1775764894.795666:0:68306:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764894.795675:0:68306:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764894.795682:0:68306:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764894.795691:0:68306:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1775764894.795698:0:68306:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764894.795702:0:68306:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764894.795704:0:68306:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c3c7 00000080:00010000:2.0:1775764894.795707:0:68306:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3c7 from it ffffa60b01fafb58 00010000:00010000:2.0:1775764894.795710:0:68306:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1775764894.795717:0:68306:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1775764894.795725:0:68306:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764894.795741:0:68306:0:(namei.c:1332:ll_lookup_nd()) VFS Op:name=f28.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), flags=8 00000080:00200000:2.0:1775764894.795745:0:68306:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f28.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b238990cb08), intent=getattr 00000002:00010000:2.0:1775764894.795758:0:68306:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f28.sanity-pcc,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764894.795806:0:68306:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68306 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.795813:0:68306:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68306 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.795822:0:68306:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68306 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764894.795850:0:68306:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:68306:1862022098737792:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764894.796008:0:68306:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85d4a80 going to sleep for 11 seconds 00000100:00100000:2.0:1775764894.800736:0:68306:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:68306:1862022098737792:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764894.800762:0:68306:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.800778:0:68306:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.800787:0:68306:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 4/1,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.800797:0:68306:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.800808:0:68306:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.800820:0:68306:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.800827:0:68306:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 5/2,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.800834:0:68306:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764894.800844:0:68306:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23b9ea4e00 x1862022098737792/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 584/640 e 0 to 0 dl 1775764910 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764894.800855:0:68306:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764894.800867:0:68306:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.800946:0:68306:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 4 res 8589949620/36 (0 0) 00000080:00200000:2.0:1775764894.800952:0:68306:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764894.801044:0:68306:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0x24:0x0], updating i_size 22 00000080:00200000:2.0:1775764894.801179:0:68306:0:(namei.c:147:ll_iget()) got inode: [0x200003ab4:0x24:0x0](ffff8b2389a642c8): rc = 0 00000080:00200000:2.0:1775764894.801212:0:68306:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x24:0x0]: layout version change: 1 -> 1 00000080:00010000:2.0:1775764894.801222:0:68306:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x24:0x0](ffff8b2389a642c8) for lock 0x73cfd86cfc17c3e3 00000080:00010000:2.0:1775764894.801234:0:68306:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x24:0x0](ffff8b2389a642c8) 00000080:00010000:2.0:1775764894.801237:0:68306:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3e3 from it ffffa60b01fafb20 00010000:00010000:2.0:1775764894.801241:0:68306:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.801251:0:68306:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1775764894.801288:0:68306:0:(file.c:6226:ll_getattr_dentry()) START file f28.sanity-pcc:[0x200003ab4:0x24:0x0](ffff8b2389a642c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1775764894.801294:0:68306:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200003ab4:0x24:0x0](ffff8b2389a642c8),name=f28.sanity-pcc 00000002:00010000:2.0:1775764894.801313:0:68306:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x24:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764894.801324:0:68306:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764894.801334:0:68306:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 2/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764894.801340:0:68306:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764894.801344:0:68306:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x24:0x0](ffff8b2389a642c8) 00000080:00010000:2.0:1775764894.801348:0:68306:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x24:0x0](ffff8b2389a642c8) for lock 0x73cfd86cfc17c3e3 00000080:00010000:2.0:1775764894.801351:0:68306:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3e3 from it ffffa60b01fafcb8 00010000:00010000:2.0:1775764894.801354:0:68306:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764894.801361:0:68306:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764894.801419:0:68306:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:2.0:1775764894.801426:0:68306:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764894.801432:0:68306:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764894.801444:0:68306:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1775764894.801450:0:68306:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764894.801696:0:68306:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:2.0:1775764894.801699:0:68306:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c3c7 00000080:00010000:2.0:1775764894.801703:0:68306:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3c7 from it ffffa60b01fafbf8 00010000:00010000:2.0:1775764894.801706:0:68306:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1775764894.801712:0:68306:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1775764894.801719:0:68306:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 1 00000080:00200000:2.0:1775764894.801726:0:68306:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f28.sanity-pcc, flags=265 00000080:00200000:2.0:1775764894.801731:0:68306:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x24:0x0](ffff8b2389a642c8), inode mode 81a4 mask 46 00000080:00200008:2.0:1775764894.801738:0:68306:0:(file.c:1057:ll_file_open()) START file f28.sanity-pcc:[0x200003ab4:0x24:0x0](ffff8b2389a642c8), flags 154402 00000002:00010000:2.0:1775764894.801753:0:68306:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200003ab4:0x24:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240114403 00010000:00010000:2.0:1775764894.801795:0:68306:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17c3ea lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68306 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.801801:0:68306:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17c3ea lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68306 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764894.801811:0:68306:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b2389398000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764894.801818:0:68306:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea4e00 x1862022098738048/t0(0) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/132264 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764894.801831:0:68306:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17c3ea lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68306 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764894.801852:0:68306:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:68306:1862022098738048:192.168.202.128@tcp:101:lfs.0 00000100:00100000:2.0:1775764894.802036:0:68306:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85d4a80 going to sleep for 11 seconds 00000100:00100000:1.0:1775764894.806279:0:46139:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119093888 00000100:00100000:1.0:1775764894.806290:0:46139:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119093888 00000100:00100000:1.0:1775764894.806344:0:46139:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 321 00000100:00100000:1.0:1775764894.806354:0:46139:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dad180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119093888:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764894.806372:0:46139:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764894.806450:0:46139:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764894.806488:0:46139:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dad180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42324:x1862022119093888:12345-192.168.202.128@tcp:104: Request processed in 136us (308us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764894.806494:0:46139:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 321 00010000:00010000:2.0:1775764894.807076:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764894.807085:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764894.807094:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2380030200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764894.807099:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764894.807108:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764894.807116:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764894.807144:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c3dc lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x24:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053387 expref: -99 pid: 68290 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1775764894.807190:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764894.807197:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764894.807221:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764894.807233:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380030200/0x73cfd86cfc17c3e3 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e030533b8 expref: -99 pid: 68306 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764894.807247:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764894.807248:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764894.807250:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764894.817114:0:68306:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:68306:1862022098738048:192.168.202.128@tcp:101:lfs.0 00010000:00010000:2.0:1775764894.817145:0:68306:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CW ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17c3ea lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533c6 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.817161:0:68306:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17c3ea lrc: 4/1,0 mode: --/CW res: [0x200003ab4:0x24:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533c6 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.817169:0:68306:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17c3ea lrc: 4/1,0 mode: --/CW res: [0x200003ab4:0x24:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533c6 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.817181:0:68306:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17c3ea lrc: 4/1,0 mode: CW/CW res: [0x200003ab4:0x24:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030533c6 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.817192:0:68306:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17c3ea lrc: 4/1,0 mode: CW/CW res: [0x200003ab4:0x24:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533c6 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.817207:0:68306:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17c3ea lrc: 5/1,1 mode: CW/CW res: [0x200003ab4:0x24:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030533c6 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.817216:0:68306:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17c3ea lrc: 5/1,1 mode: CW/CW res: [0x200003ab4:0x24:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030533c6 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.817223:0:68306:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17c3ea lrc: 4/0,1 mode: CW/CW res: [0x200003ab4:0x24:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030533c6 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764894.817234:0:68306:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b23b9ea4e00 x1862022098738048/t34359739467(34359739467) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764910 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764894.817258:0:68306:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4e00 x1862022098738048/t34359739467(34359739467) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 1056/664 e 0 to 0 dl 1775764910 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764894.817276:0:68306:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17c3ea lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0x24:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533c6 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.817287:0:68306:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2398249800 type 13 mode 8 res 8589949620/36 (0 0) 00000080:00200000:2.0:1775764894.817294:0:68306:0:(namei.c:953:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764894.817385:0:68306:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0x24:0x0], updating i_size 22 00000080:00010000:2.0:1775764894.817391:0:68306:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x24:0x0](ffff8b2389a642c8) for lock 0x73cfd86cfc17c3ea 00000080:00010000:2.0:1775764894.817396:0:68306:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3ea from it ffffa60b01fafbf0 00010000:00010000:2.0:1775764894.817400:0:68306:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17c3ea lrc: 3/0,1 mode: CW/CW res: [0x200003ab4:0x24:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030533c6 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.817408:0:68306:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17c3ea lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x24:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533c6 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:2.0:1775764894.817433:0:68306:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea4e00 x1862022098738048/t34359739467(34359739467) o101->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 576/664 e 0 to 0 dl 1775764910 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1775764894.817563:0:68306:0:(file.c:4602:ll_file_ioctl()) VFS Op:inode=[0x200003ab4:0x24:0x0](ffff8b2389a642c8) cmd=c00466fc arg=7ffc1671e47c 00000080:00200008:2.0:1775764894.817587:0:68306:0:(file.c:447:ll_file_release()) START file f28.sanity-pcc:[0x200003ab4:0x24:0x0](ffff8b2389a642c8), flags 154002 00010000:00010000:2.0:1775764894.817607:0:68306:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2380031800/0x73cfd86cfc17c3ea lrc: 2/0,0 mode: CW/CW res: [0x200003ab4:0x24:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533c6 expref: -99 pid: 68306 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1775764894.904709:0:68316:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08),name=/ 00000002:00010000:1.0:1775764894.904741:0:68316:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764894.904761:0:68316:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764894.904778:0:68316:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1775764894.904787:0:68316:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1775764894.904795:0:68316:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) 00000080:00010000:1.0:1775764894.904799:0:68316:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b238990cb08) for lock 0x73cfd86cfc17c3c7 00000080:00010000:1.0:1775764894.904806:0:68316:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3c7 from it ffffa60b01fd7bb0 00010000:00010000:1.0:1775764894.904811:0:68316:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1775764894.904819:0:68316:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740c00/0x73cfd86cfc17c3c7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e03053348 expref: -99 pid: 68269 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:1.0:1775764894.904836:0:68316:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08), inode mode 41ed mask 44 00000080:00200008:1.0:1775764894.904843:0:68316:0:(file.c:1057:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200080:1.0:1775764894.904867:0:68316:0:(dir.c:1998:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b238990cb08) cmd=901066fc arg=7ffe5c710140 00000100:00100000:1.0:1775764894.904934:0:68316:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:68316:1862022098738304:192.168.202.128@tcp:34:lfs.0 00000100:00100000:1.0:1775764894.905176:0:68316:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654840 going to sleep for 11 seconds 00000100:00100000:2.0:1775764894.911331:0:68316:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job lfs:fcf15866-4739-4595-8fe1-44abde7ee9d7:68316:1862022098738304:192.168.202.128@tcp:34:lfs.0 00000080:00200008:2.0:1775764894.911407:0:68316:0:(file.c:447:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b238990cb08), flags 304000 00000080:00200000:1.0:1775764894.965232:0:68318:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:1.0:1775764894.965262:0:68318:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1775764894.965336:0:68318:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68318 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.965347:0:68318:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68318 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1775764894.965358:0:68318:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68318 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1775764894.965386:0:68318:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job multiop:82781e18-e0e3-4f9f-9a2b-041bf0344414:68318:1862022098738432:192.168.202.128@tcp:101:multiop.0 00000100:00100000:1.0:1775764894.965519:0:68318:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b238b654540 going to sleep for 11 seconds 00000100:00100000:2.0:1775764894.973804:0:68318:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job multiop:82781e18-e0e3-4f9f-9a2b-041bf0344414:68318:1862022098738432:192.168.202.128@tcp:101:multiop.0 00010000:00010000:2.0:1775764894.973831:0:68318:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533f0 expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.973845:0:68318:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533f0 expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.973857:0:68318:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030533f0 expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.973868:0:68318:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533f0 expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.973881:0:68318:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030533f0 expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.973889:0:68318:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030533f0 expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.973896:0:68318:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030533f0 expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764894.973915:0:68318:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b23864fc380 x1862022098738432/t0(0) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 576/688 e 0 to 0 dl 1775764910 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764894.973928:0:68318:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533f0 expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.973940:0:68318:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2387c07800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1775764894.973947:0:68318:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1775764894.973974:0:68318:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 9728 00000080:00010000:2.0:1775764894.973980:0:68318:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17c3f1 00000080:00010000:2.0:1775764894.973985:0:68318:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:2.0:1775764894.973988:0:68318:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3f1 from it ffffa60b01ed3bf8 00010000:00010000:2.0:1775764894.973991:0:68318:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030533f0 expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.973999:0:68318:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533f0 expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764894.974033:0:68318:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148), inode mode 41ed mask 1 00000080:00200000:2.0:1775764894.974058:0:68318:0:(file.c:6121:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148),name=/ 00000002:00010000:2.0:1775764894.974072:0:68318:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1775764894.974080:0:68318:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533f0 expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764894.974090:0:68318:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533f0 expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764894.974101:0:68318:0:(namei.c:953:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1775764894.974106:0:68318:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) 00000080:00010000:2.0:1775764894.974108:0:68318:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b2389909148) for lock 0x73cfd86cfc17c3f1 00000080:00010000:2.0:1775764894.974112:0:68318:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3f1 from it ffffa60b01ed3b38 00010000:00010000:2.0:1775764894.974114:0:68318:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e030533f0 expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764894.974121:0:68318:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030533f0 expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775764894.974128:0:68318:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b2389909148), inode mode 41ed mask 3 00000080:00200000:2.0:1775764894.974133:0:68318:0:(namei.c:1409:ll_atomic_open()) VFS Op:name=f28.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2389909148), file ffff8b2392e20300, open_flags 8042, mode 81a4 opened 0 00000080:00200000:2.0:1775764894.974142:0:68318:0:(namei.c:1073:ll_lookup_it()) VFS Op:name=f28.sanity-pcc, dir=[0x200000007:0x1:0x0](ffff8b2389909148), intent=open|creat 00000002:00010000:2.0:1775764894.974162:0:68318:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f28.sanity-pcc,[0x200003ab2:0x2:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0100103 00010000:00010000:2.0:1775764894.974175:0:68318:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e030533f0 expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764894.974221:0:68318:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030533f0 expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764894.974228:0:68318:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764894.974231:0:68318:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b23b8741e00/0x73cfd86cfc17c3f1 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030533f0 expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764894.974265:0:68318:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68318 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.974277:0:68318:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68318 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1775764894.974287:0:68318:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b238a3bf000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1775764894.974294:0:68318:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7480 x1862022098738688/t0(0) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 1064/66728 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1775764894.974304:0:68318:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68318 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1775764894.974328:0:68318:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job multiop:82781e18-e0e3-4f9f-9a2b-041bf0344414:68318:1862022098738688:192.168.202.128@tcp:101:multiop.0 00000100:00100000:2.0:1775764894.974517:0:68318:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85d4240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764894.984741:0:68318:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job multiop:82781e18-e0e3-4f9f-9a2b-041bf0344414:68318:1862022098738688:192.168.202.128@tcp:101:multiop.0 00010000:00010000:2.0:1775764894.984769:0:68318:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.984795:0:68318:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.984802:0:68318:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 4/0,1 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.984812:0:68318:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.984821:0:68318:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 4/0,1 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.984838:0:68318:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.984845:0:68318:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 5/1,1 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.984851:0:68318:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 4/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1775764894.984860:0:68318:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=2b, status=0 req@ffff8b23b9ea7480 x1862022098738688/t34359739468(34359739468) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764910 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1775764894.984871:0:68318:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7480 x1862022098738688/t34359739468(34359739468) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 1064/664 e 0 to 0 dl 1775764910 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1775764894.984883:0:68318:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1775764894.984895:0:68318:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.984911:0:68318:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff8b2387c07800 type 13 mode 4 res 8589949620/36 (0 0) 00000080:00200000:2.0:1775764894.984917:0:68318:0:(namei.c:953:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1775764894.984951:0:68318:0:(llite_lib.c:3177:ll_read_inode2()) VFS Op:inode=[0x200003ab4:0x24:0x0](ffff8b2390ec6c08) 00000080:00200000:2.0:1775764894.985032:0:68318:0:(llite_lib.c:2941:ll_update_inode()) inode=[0x200003ab4:0x24:0x0], updating i_size 22 00000080:00200000:2.0:1775764894.985062:0:68318:0:(file.c:6766:ll_layout_conf()) [0x200003ab4:0x24:0x0]: layout version change: 4294967294 -> 1 00000080:00010000:2.0:1775764894.985073:0:68318:0:(llite_internal.h:1984:ll_set_lock_data()) setting l_data to inode [0x200003ab4:0x24:0x0](ffff8b2390ec6c08) for lock 0x73cfd86cfc17c3f8 00000080:00010000:2.0:1775764894.985085:0:68318:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200003ab4:0x24:0x0](ffff8b2390ec6c08) 00000080:00200008:2.0:1775764894.985100:0:68318:0:(file.c:1057:ll_file_open()) START file f28.sanity-pcc:[0x200003ab4:0x24:0x0](ffff8b2390ec6c08), flags 100102 00000002:00100000:2.0:1775764894.985120:0:68318:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b23b9ea7480 x1862022098738688/t34359739468(34359739468) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764910 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1775764894.985302:0:68318:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x73cfd86cfc17c3f8 from it ffff8b238504c540 00010000:00010000:2.0:1775764894.985306:0:68318:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 3/1,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1775764894.985315:0:68318:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764894.985333:0:68318:0:(file.c:6593:ll_inode_permission()) VFS Op:inode=[0x200003ab4:0x24:0x0](ffff8b2390ec6c08), inode mode 81a4 mask 46 00000100:00100000:1.0:1775764895.047356:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00000100:00100000:2.0:1775764895.053224:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098739200:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764895.057783:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098739200:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764895.064211:0:2369:0:(events.c:340:request_in_callback()) peer: 12345-192.168.202.128@tcp (source: 12345-192.168.202.128@tcp) 00010000:00010000:2.0:1775764895.064526:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764895.064533:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c3ff lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0305342f expref: -99 pid: 68325 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764895.064542:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2381497200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764895.064546:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c3ff lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e0305342f expref: -99 pid: 68325 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764895.064650:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c3ff lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e0305342f expref: -99 pid: 68325 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764895.064658:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c3ff lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e0305342f expref: -99 pid: 68325 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764895.064690:0:46149:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381496a00/0x73cfd86cfc17c3dc lrc: 2/0,0 mode: CR/CR res: [0x200003ab4:0x24:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x57b8ff0e03053387 expref: -99 pid: 68290 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1775764895.064732:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c3ff lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e0305342f expref: -99 pid: 68325 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764895.064739:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764895.064878:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c3ff lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e0305342f expref: -99 pid: 68325 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764895.064886:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497200/0x73cfd86cfc17c3ff lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e0305342f expref: -99 pid: 68325 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775764895.064902:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764895.064904:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764895.064911:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764895.067699:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098739584:192.168.202.128@tcp:103:ldlm_bl.0 00000080:00200008:2.0:1775764895.096542:0:68318:0:(file.c:447:ll_file_release()) START file f28.sanity-pcc:[0x200003ab4:0x24:0x0](ffff8b2390ec6c08), flags 100002 00000002:00100000:2.0:1775764895.096614:0:68318:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff8b23b9ea7480 x1862022098738688/t34359739468(34359739468) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764910 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1775764895.096626:0:68318:0:(genops.c:2442:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b238a3bf000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1775764895.096631:0:68318:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b23b9ea7800 x1862022098740352/t0(0) o35->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:23/10 lens 392/1032 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1775764895.096655:0:68318:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job multiop:82781e18-e0e3-4f9f-9a2b-041bf0344414:68318:1862022098740352:192.168.202.128@tcp:35:multiop.0 00000100:00100000:2.0:1775764895.096748:0:68318:0:(client.c:2691:ptlrpc_set_wait()) set ffff8b23b85d4240 going to sleep for 11 seconds 00000100:00100000:2.0:1775764895.102058:0:68318:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job multiop:82781e18-e0e3-4f9f-9a2b-041bf0344414:68318:1862022098740352:192.168.202.128@tcp:35:multiop.0 00000002:00100000:2.0:1775764895.102095:0:68318:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b23b9ea7480 x1862022098738688/t34359739468(34359739468) o101->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 584/664 e 0 to 0 dl 1775764910 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1775764895.188091:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119094272 00000100:00100000:1.0:1775764895.188102:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119094272 00000100:00100000:1.0:1775764895.188163:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 324 00000100:00100000:1.0:1775764895.188174:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382dadc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42326:x1862022119094272:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764895.188193:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b2381497600/0x73cfd86cfc17c414 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0305346e expref: -99 pid: 68331 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1775764895.188279:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764895.188286:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382dadc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:fcf15866-4739-4595-8fe1-44abde7ee9d7+5:42326:x1862022119094272:12345-192.168.202.128@tcp:104: Request processed in 113us (577us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764895.188293:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 324 00000100:00100000:2.0:1775764895.190934:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098740864:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764895.199075:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2379:1862022098740864:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1775764895.212830:0:46140:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1862022119094400 00000100:00100000:1.0:1775764895.212841:0:46140:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1862022119094400 00000100:00100000:1.0:1775764895.212938:0:46140:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.202.128@tcp, seq: 325 00000100:00100000:1.0:1775764895.212949:0:46140:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff8b2382daf800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42325:x1862022119094400:12345-192.168.202.128@tcp:104: 00010000:00010000:1.0:1775764895.212966:0:46140:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1775764895.213054:0:46140:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1775764895.213062:0:46140:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff8b2382daf800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:82781e18-e0e3-4f9f-9a2b-041bf0344414+5:42325:x1862022119094400:12345-192.168.202.128@tcp:104: Request processed in 115us (362us total) trans 0 rc 0/0 00000100:00100000:1.0:1775764895.213069:0:46140:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.202.128@tcp, seq: 325 00010000:00010000:2.0:1775764895.213122:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775764895.213130:0:46149:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764895.213139:0:46149:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff8b2380030200 already unused, calling callback (ffffffffc19ed2e0) 00010000:00010000:2.0:1775764895.213144:0:46149:0:(ldlm_request.c:1724:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x1a rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764895.213151:0:46149:0:(ldlm_request.c:1734:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x1a rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764895.213159:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 3/0,0 mode: PR/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x1a rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1775764895.213204:0:46149:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x24:0x0]: losing layout lock 00010000:00010000:2.0:1775764895.213260:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 2/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x1a rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764895.213267:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1775764895.213297:0:46149:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x1a rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764895.213309:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b238a3bf000 lock: ffff8b2380030200/0x73cfd86cfc17c3f8 lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x24:0x0].0x0 bits 0x1a/0x1a rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x57b8ff0e0305340c expref: -99 pid: 68318 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775764895.213331:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764895.213333:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764895.213335:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764895.213343:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764895.213358:0:2379:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2379:1862022098741632:192.168.202.128@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1775764895.219825:0:2379:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2379:1862022098741632:192.168.202.128@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1775764896.695788:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764898.743606:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764900.791594:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764902.839604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764903.991649:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2376:1862022098742528:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764903.991729:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098742784:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764903.991768:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098743040:192.168.202.128@tcp:400:kworker.0 00010000:00010000:2.0:1775764904.887604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764906.935583:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764908.983604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764909.111630:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2376:1862022098743296:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764909.111710:0:2376:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2376:1862022098743552:192.168.202.128@tcp:400:kworker.0 00010000:00010000:2.0:1775764911.031587:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764913.079602:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1775764914.231567:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764898/real 1775764898] req@ffff8b23864f1f80 x1862022098741888/t0(0) o400->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764914.239026:0:2378:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b23864f1f80 x1862022098741888/t0(0) o400->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764914.239037:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1775764914.239041:0:2378:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.202.128@tcp: Connection to MGS (at 192.168.202.128@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1775764914.242508:0:2378:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.202.128@tcp for MGC192.168.202.128@tcp not replayable, auto-deactivating 00000100:00080000:2.0:1775764914.242512:0:2378:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:2.0:1775764914.242519:0:2378:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b23864f1f80 x1862022098741888/t0(0) o400->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764914.242533:0:2378:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b2382daed80 x1862022098742528/t0(0) o400->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764919 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764914.242553:0:2378:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b23864fdf80 x1862022098743168/t0(0) o400->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764925 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764914.242568:0:2378:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b23864f2d80 x1862022098743808/t0(0) o400->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764930 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:2.0:1775764914.242581:0:2378:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1775764914.242584:0:2378:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:2.0:1775764914.242586:0:2378:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 00000100:00080000:2.0:1775764914.242588:0:2378:0:(import.c:416:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000400:2.0:1775764914.242623:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764898/real 1775764898] req@ffff8b23864f0380 x1862022098742144/t0(0) o400->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764914.242649:0:2378:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b23864f0380 x1862022098742144/t0(0) o400->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764914.242666:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1775764914.242673:0:2378:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff8b2389398000: Connection to lustre-MDT0000 (at 192.168.202.128@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:2.0:1775764914.246943:0:2378:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00000400:2.0:1775764914.246971:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764898/real 1775764898] req@ffff8b23864f0a80 x1862022098742400/t0(0) o400->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764914.246992:0:2378:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b23864f0a80 x1862022098742400/t0(0) o400->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764914.247005:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1775764914.247012:0:2378:0:(import.c:181:ptlrpc_set_import_discon()) lustre-OST0001-osc-ffff8b2389398000: Connection to lustre-OST0001 (at 192.168.202.128@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:2.0:1775764914.247016:0:2378:0:(import.c:416:ptlrpc_pinger_force()) lustre-OST0001_UUID: waking up pinger s:DISCONN 00000100:00000400:2.0:1775764914.247037:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764898/real 1775764898] req@ffff8b23864f1c00 x1862022098742016/t0(0) o400->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764914.247051:0:2378:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b23864f1c00 x1862022098742016/t0(0) o400->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764914.247065:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1775764914.247073:0:2378:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff8b238a3bf000: Connection to lustre-MDT0000 (at 192.168.202.128@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:2.0:1775764914.247077:0:2378:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00000400:2.0:1775764914.247094:0:2378:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764898/real 1775764898] req@ffff8b23864f1180 x1862022098742272/t0(0) o400->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764914.247106:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098744576:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775764914.247119:0:2378:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b23864f1180 x1862022098742272/t0(0) o400->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764914 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764914.247130:0:2378:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1775764914.247135:0:2378:0:(import.c:181:ptlrpc_set_import_discon()) lustre-OST0000-osc-ffff8b2389398000: Connection to lustre-OST0000 (at 192.168.202.128@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:2.0:1775764914.247138:0:2378:0:(import.c:416:ptlrpc_pinger_force()) lustre-OST0000_UUID: waking up pinger s:DISCONN 00000100:00100000:2.0:1775764914.247157:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:24b54882-c0a1-43f2-954f-009e5a9afc9d:2378:1862022098741888:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764914.247170:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098742144:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764914.247176:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098744704:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775764914.247182:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098742400:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764914.247194:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:82781e18-e0e3-4f9f-9a2b-041bf0344414:2378:1862022098742016:192.168.202.128@tcp:400:kworker.0 00000100:00100000:2.0:1775764914.247205:0:2378:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fcf15866-4739-4595-8fe1-44abde7ee9d7:2378:1862022098742272:192.168.202.128@tcp:400:kworker.0 00010000:00010000:2.0:1775764915.127550:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764917.175548:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764919.223589:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1775764919.351560:0:2377:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764903/real 1775764903] req@ffff8b2382daf100 x1862022098742656/t0(0) o400->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764919 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764919.360652:0:2377:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b2382daf100 x1862022098742656/t0(0) o400->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764919 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764919.360660:0:2377:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b23a00b1000 already not connected (conn 2, was 1): CONNECTING 00000100:00000400:1.0:1775764919.360665:0:2377:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764903/real 1775764903] req@ffff8b2382dac000 x1862022098742912/t0(0) o400->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764919 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764919.360673:0:2377:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b2382dac000 x1862022098742912/t0(0) o400->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764919 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764919.360677:0:2377:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff8b2384b87000 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:1.0:1775764919.360686:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:82781e18-e0e3-4f9f-9a2b-041bf0344414:2377:1862022098742656:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764919.360689:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098742912:192.168.202.128@tcp:400:kworker.0 00010000:00010000:2.0:1775764921.271554:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764923.319569:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764924.471531:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764914/real 1775764914] req@ffff8b2382dae680 x1862022098744960/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764924.471567:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098744960/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764924.471589:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764914/real 1775764914] req@ffff8b2382dad500 x1862022098744832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764924.471603:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098744832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764924.471612:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764914/real 1775764914] req@ffff8b2382daea00 x1862022098744704/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764924.471632:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098744704/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764924.471641:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764914/real 1775764914] req@ffff8b2382daed80 x1862022098744576/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764924.471652:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098744576/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764924.471667:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764914/real 1775764914] req@ffff8b23864f0700 x1862022098744448/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764924.471678:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098744448/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764924 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764924.471702:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764924.471718:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764924.471722:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098744960:192.168.202.128@tcp:8:kworker.0 00000100:00080000:2.0:1775764924.471728:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764924.471733:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764924.471736:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098744832:192.168.202.128@tcp:38:kworker.0 00000100:00080000:2.0:1775764924.471741:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764924.471761:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764924.471765:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098744704:192.168.202.128@tcp:8:kworker.0 00000100:00080000:2.0:1775764924.471769:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764924.471775:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764924.471778:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098744576:192.168.202.128@tcp:38:kworker.0 00000100:00080000:2.0:1775764924.471789:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764924.471809:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764924.471813:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098744448:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775764924.471932:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098745088:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775764924.472015:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098745216:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775764924.472037:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098745344:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775764924.472075:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098745472:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775764924.472128:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098745600:192.168.202.128@tcp:250:kworker.0 00010000:00010000:2.0:1775764925.367565:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1775764925.495550:0:2377:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764909/real 1775764909] req@ffff8b23864fc380 x1862022098743424/t0(0) o400->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764925 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764925.505481:0:2377:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b23864fc380 x1862022098743424/t0(0) o400->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764925 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764925.505490:0:2377:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b2384b86000 already not connected (conn 3, was 1): CONNECTING 00000100:00000400:1.0:1775764925.505496:0:2377:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764909/real 1775764909] req@ffff8b23864fce00 x1862022098743680/t0(0) o400->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764925 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764925.505504:0:2377:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b23864fce00 x1862022098743680/t0(0) o400->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 224/224 e 0 to 1 dl 1775764925 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764925.505509:0:2377:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff8b2384b83000 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:1.0:1775764925.505518:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098743424:192.168.202.128@tcp:400:kworker.0 00000100:00100000:1.0:1775764925.505522:0:2377:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fcf15866-4739-4595-8fe1-44abde7ee9d7:2377:1862022098743680:192.168.202.128@tcp:400:kworker.0 00010000:00010000:2.0:1775764927.415588:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764929.463560:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764931.511621:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764933.559598:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764935.607658:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764937.655680:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764939.703660:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764941.751611:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764943.799588:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775764944.951621:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764944.951631:0:10503:0:(pinger.c:247:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764944.951634:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764944.951638:0:10503:0:(pinger.c:247:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764944.951641:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764944.951643:0:10503:0:(pinger.c:247:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764944.951645:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764944.951648:0:10503:0:(pinger.c:247:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764944.951651:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764944.951653:0:10503:0:(pinger.c:247:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1775764945.847652:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764947.895630:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764949.943613:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775764950.071638:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764950.071651:0:10503:0:(pinger.c:247:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764950.071655:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764950.071659:0:10503:0:(pinger.c:247:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764950.071663:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764950.071667:0:10503:0:(pinger.c:247:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764950.071670:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764950.071674:0:10503:0:(pinger.c:247:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764950.071677:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764950.071681:0:10503:0:(pinger.c:247:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1775764951.991608:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764954.039625:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764956.087544:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:2.0:1775764957.303554:0:2367:0:(socklnd_cb.c:2376:ksocknal_find_timed_out_conn()) A connection with 12345-192.168.202.128@tcp (192.168.202.128:988) timed out; the network or node may be down. 00010000:00010000:2.0:1775764958.135551:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775764959.287775:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764959.287785:0:10503:0:(pinger.c:247:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1775764959.287789:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764959.287793:0:10503:0:(pinger.c:247:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1775764959.287797:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764959.287801:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775764959.287805:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764959.287817:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2406 00000100:00080000:1.0:1775764959.287834:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1775764959.287843:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775764959.287861:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764959.287904:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098746368/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764959.287928:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764959.287932:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775764959.287936:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764959.287942:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2406 00000100:00080000:1.0:1775764959.287947:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1775764959.287951:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775764959.287957:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764959.287973:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098746496/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764959.287986:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764959.287992:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775764959.287996:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764959.288006:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2406 00000100:00080000:1.0:1775764959.288010:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1775764959.288014:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775764959.288020:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764959.288034:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098746624/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764959.288051:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764959.288055:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775764959.288057:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764959.288061:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2406 00000100:00080000:1.0:1775764959.288066:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1775764959.288071:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775764959.288090:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098746752/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764959.288106:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764959.288110:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775764959.288112:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764959.288116:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2406 00000100:00080000:1.0:1775764959.288120:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1775764959.288125:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775764959.288130:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764959.288144:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098746880/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775764960.183619:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764962.231576:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775764962.359782:0:2364:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.202.128:988 00000400:00000100:2.0:1775764962.359814:0:2364:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775764962.359825:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775764962.359860:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775764962.359868:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775764962.359874:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775764962.359882:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00010000:00010000:2.0:1775764964.279536:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775764965.431708:0:2365:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.202.128:988 00000400:00000100:1.0:1775764965.431755:0:2365:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775764965.431772:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775764965.431810:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775764965.431818:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775764965.431831:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775764965.431837:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00010000:00010000:2.0:1775764966.327614:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764968.375611:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764970.423620:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764972.471561:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775764972.599786:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.202.128:988 00000400:00000100:2.0:1775764972.599847:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775764972.599870:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775764972.599926:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775764972.599938:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775764972.599948:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775764972.599958:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00010000:00010000:2.0:1775764974.519578:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764976.567593:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764978.615554:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775764979.767613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764979.767622:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775764979.767626:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764979.767635:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2441 00000100:00080000:2.0:1775764979.767646:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775764979.767652:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775764979.767686:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7800 x1862022098748928/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775764979.767716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764979.767720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775764979.767722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764979.767725:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2441 00000100:00080000:2.0:1775764979.767728:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775764979.767733:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775764979.767742:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764979.767757:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098749056/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775764979.767769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764979.767772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775764979.767774:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764979.767779:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2441 00000100:00080000:2.0:1775764979.767791:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775764979.767796:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775764979.767801:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764979.767811:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022098749184/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775764979.767821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764979.767825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775764979.767828:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764979.767833:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2441 00000100:00080000:2.0:1775764979.767837:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775764979.767841:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775764979.767846:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764979.767860:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098749312/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775764979.767872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764979.767875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775764979.767877:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764979.767881:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2441 00000100:00080000:2.0:1775764979.767886:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775764979.767890:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775764979.767897:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764979.767913:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098749440/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775764980.663551:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764982.711579:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764982.839922:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764979/real 1775764982] req@ffff8b23b9ea7100 x1862022098749312/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764982.839955:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022098749312/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764982.839968:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764982.839974:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764982.839979:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098749312:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775764982.839989:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764979/real 1775764982] req@ffff8b23b9ea4e00 x1862022098749440/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764982.840001:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022098749440/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764982.840009:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764982.840013:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764982.840017:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098749440:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775764982.840024:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764979/real 1775764982] req@ffff8b23b9ea7480 x1862022098749056/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764982.840038:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022098749056/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764982.840046:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764982.840050:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764982.840053:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098749056:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775764982.840060:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764979/real 1775764982] req@ffff8b23b9ea6300 x1862022098749184/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764982.840070:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022098749184/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764982.840076:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764982.840079:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764982.840082:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098749184:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775764982.840088:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764979/real 1775764982] req@ffff8b23b9ea7800 x1862022098748928/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764982.840100:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7800 x1862022098748928/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764982.840107:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764982.840110:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764982.840113:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098748928:192.168.202.128@tcp:250:kworker.0 00010000:00010000:2.0:1775764984.759559:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764984.887713:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098749568:192.168.202.128@tcp:250:kworker.0 00000800:00000100:2.0:1775764984.887784:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775764984.887848:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764984/real 1775764984] req@ffff8b2382dae680 x1862022098749568/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764984.887874:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098749568/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764984.887888:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764984.887892:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764984.887897:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098749568:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775764984.887927:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098749696:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775764984.887950:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775764984.887963:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764984/real 1775764984] req@ffff8b2382dad500 x1862022098749696/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764984.887976:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098749696/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764984.887984:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764984.887988:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764984.887992:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098749696:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775764984.887998:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098749824:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775764984.888016:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775764984.888027:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764984/real 1775764984] req@ffff8b2382daea00 x1862022098749824/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764984.888054:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098749824/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764984.888062:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764984.888066:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764984.888069:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098749824:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775764984.888075:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098749952:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775764984.888097:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775764984.888108:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764984/real 1775764984] req@ffff8b2382daed80 x1862022098749952/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764984.888118:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098749952/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764984.888126:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764984.888128:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764984.888132:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098749952:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775764984.888138:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098750080:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775764984.888155:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775764984.888165:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764984/real 1775764984] req@ffff8b2382dace00 x1862022098750080/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764984.888175:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098750080/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764984.888183:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764984.888186:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764984.888200:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098750080:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775764986.807567:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764988.855571:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775764990.007567:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764990.007579:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775764990.007584:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764990.007597:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2451 00000100:00080000:1.0:1775764990.007610:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764990.007617:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775764990.007662:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098750208/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764990.007700:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764990.007708:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775764990.007710:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764990.007714:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2451 00000100:00080000:1.0:1775764990.007719:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764990.007723:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775764990.007735:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764990.007747:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098750336/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764990.007757:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764990.007760:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775764990.007763:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764990.007771:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2451 00000100:00100000:2.0:1775764990.007772:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098750208:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775764990.007775:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764990.007785:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775764990.007796:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764990.007806:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098750464/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764990.007812:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764990.007816:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775764990.007818:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764990.007823:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2451 00000100:00080000:1.0:1775764990.007829:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764990.007834:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775764990.007840:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764990.007856:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098750592/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764990.007866:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764990.007870:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775764990.007876:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775764990.007877:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775764990.007897:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2451 00000100:00080000:1.0:1775764990.007902:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764990.007908:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775764990.007915:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764990.007934:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098750720/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764990.007946:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764990/real 1775764990] req@ffff8b23864fd180 x1862022098750208/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764990.007969:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098750208/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764990.007989:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764990.007994:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764990.007999:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098750208:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775764990.008039:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098750336:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775764990.008071:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775764990.008083:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764990/real 1775764990] req@ffff8b23864fea00 x1862022098750336/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764990.008095:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098750336/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764990.008103:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764990.008107:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764990.008110:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098750336:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775764990.008123:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098750464:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775764990.008187:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775764990.008198:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764990/real 1775764990] req@ffff8b23864ffb80 x1862022098750464/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764990.008209:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098750464/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764990.008216:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764990.008220:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764990.008223:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098750464:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775764990.008228:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098750592:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775764990.008249:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775764990.008258:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764990/real 1775764990] req@ffff8b23864fd500 x1862022098750592/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764990.008273:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098750592/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764990.008279:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764990.008283:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764990.008286:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098750592:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775764990.008292:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098750720:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775764990.008309:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775764990.008318:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764990/real 1775764990] req@ffff8b23864ff100 x1862022098750720/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764990.008328:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098750720/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764990.008333:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764990.008337:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764990.008339:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098750720:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775764990.903554:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764992.951591:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764994.999552:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775764995.127610:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764995.127622:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775764995.127628:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764995.127640:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2456 00000100:00080000:1.0:1775764995.127653:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764995.127661:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775764995.127711:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098750848/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764995.127747:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764995.127754:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775764995.127756:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764995.127760:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2456 00000100:00080000:1.0:1775764995.127764:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764995.127769:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775764995.127780:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764995.127794:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098750976/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764995.127802:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098750848:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775764995.127810:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764995.127814:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775764995.127816:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764995.127821:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2456 00000100:00080000:1.0:1775764995.127824:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764995.127828:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775764995.127834:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764995.127848:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098751104/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764995.127858:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764995.127862:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775764995.127864:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764995.127869:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2456 00000100:00080000:1.0:1775764995.127873:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764995.127877:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775764995.127882:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764995.127894:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098751232/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764995.127908:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764995.127911:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775764995.127913:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764995.127918:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2456 00000100:00080000:1.0:1775764995.127921:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764995.127925:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775764995.127931:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764995.127931:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098750976:192.168.202.128@tcp:38:kworker.0 00000100:00100000:1.0:1775764995.127953:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098751360/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764995.127968:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098751104:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775764995.128002:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098751232:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775764995.128032:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098751360:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775764997.047869:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775764998.200024:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764995/real 1775764998] req@ffff8b23864ffb80 x1862022098751360/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764998.200063:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098751360/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764998.200108:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764998.200116:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764998.200122:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098751360:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775764998.200133:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764995/real 1775764998] req@ffff8b23864fe300 x1862022098750976/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764998.200150:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098750976/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764998.200159:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764998.200163:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764998.200166:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098750976:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775764998.200174:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764995/real 1775764998] req@ffff8b23864ff100 x1862022098751104/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764998.200188:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098751104/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764998.200197:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764998.200202:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764998.200205:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098751104:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775764998.200212:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764995/real 1775764998] req@ffff8b23864fd500 x1862022098751232/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764998.200224:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098751232/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764998.200232:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764998.200236:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764998.200240:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098751232:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775764998.200249:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764995/real 1775764998] req@ffff8b23864fdc00 x1862022098750848/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764998.200262:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098750848/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764998.200270:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775764998.200273:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775764998.200277:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098750848:192.168.202.128@tcp:250:kworker.0 00010000:00010000:2.0:1775764999.095579:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765000.247728:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098751488:192.168.202.128@tcp:250:kworker.0 00000800:00000100:2.0:1775765000.247832:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765000.247887:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765000/real 1775765000] req@ffff8b2382dac700 x1862022098751488/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765000.247908:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098751488/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765000.247934:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765000.247939:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765000.247943:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098751488:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765000.247952:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098751616:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765000.247979:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765000.247991:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765000/real 1775765000] req@ffff8b2382daf480 x1862022098751616/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765000.248003:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098751616/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765000.248010:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765000.248025:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765000.248029:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098751616:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765000.248082:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098751744:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765000.248105:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765000.248116:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765000/real 1775765000] req@ffff8b2382dadf80 x1862022098751744/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765000.248127:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098751744/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765000.248140:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765000.248143:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765000.248146:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098751744:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765000.248151:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098751872:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765000.248173:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765000.248184:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765000/real 1775765000] req@ffff8b2382dafb80 x1862022098751872/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765000.248194:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098751872/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765000.248200:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765000.248203:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765000.248205:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098751872:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765000.248210:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098752000:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765000.248227:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765000.248236:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765000/real 1775765000] req@ffff8b2382dace00 x1862022098752000/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765000.248245:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098752000/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765000.248252:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765000.248254:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765000.248258:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098752000:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765001.143604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765003.191527:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765005.239578:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765005.367557:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765005.367563:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765005.367567:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765005.367574:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2466 00000100:00080000:1.0:1775765005.367583:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765005.367591:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765005.367625:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098752128/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765005.367644:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765005.367647:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765005.367649:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765005.367652:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2466 00000100:00080000:1.0:1775765005.367656:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765005.367660:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765005.367671:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765005.367684:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098752256/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765005.367696:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765005.367700:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098752128:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775765005.367701:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765005.367707:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765005.367719:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2466 00000100:00080000:1.0:1775765005.367723:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765005.367727:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765005.367732:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765005.367745:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098752384/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765005.367751:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765005.367754:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765005.367755:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765005.367757:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2466 00000100:00080000:1.0:1775765005.367760:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765005.367762:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765005.367765:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765005.367774:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098752512/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765005.367784:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775765005.367796:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765005.367798:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765005.367800:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765005.367803:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2466 00000100:00080000:1.0:1775765005.367807:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765005.367809:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765005.367814:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765005.367820:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098752640/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765005.367853:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765005/real 1775765005] req@ffff8b23864fea00 x1862022098752128/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765005.367869:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098752128/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765005.367877:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765005.367881:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765005.367884:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098752128:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765005.367914:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098752256:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765005.367939:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765005.367953:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765005/real 1775765005] req@ffff8b23864fd180 x1862022098752256/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765005.367961:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098752256/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765005.367966:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765005.367969:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765005.367970:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098752256:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765005.367977:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098752384:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765005.367991:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765005.367997:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765005/real 1775765005] req@ffff8b23864fce00 x1862022098752384/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765005.368026:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098752384/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765005.368030:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765005.368033:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765005.368036:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098752384:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765005.368042:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098752512:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765005.368063:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765005.368073:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765005/real 1775765005] req@ffff8b23864fe680 x1862022098752512/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765005.368093:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098752512/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765005.368100:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765005.368102:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765005.368104:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098752512:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765005.368108:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098752640:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765005.368124:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765005.368130:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765005/real 1775765005] req@ffff8b23864fca80 x1862022098752640/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765005.368138:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098752640/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765005.368142:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765005.368144:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765005.368146:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098752640:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765007.287548:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765009.335581:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765010.487744:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098752768:192.168.202.128@tcp:250:kworker.0 00000800:00000100:2.0:1775765010.487823:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765010.487876:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765010/real 1775765010] req@ffff8b2382daed80 x1862022098752768/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765010.487893:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098752768/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765010.487902:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765010.487906:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765010.487909:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098752768:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765010.487914:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098752896:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765010.487931:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765010.487939:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765010/real 1775765010] req@ffff8b2382daea00 x1862022098752896/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765010.487947:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098752896/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765010.487972:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765010.487976:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765010.487979:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098752896:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765010.488011:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098753024:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765010.488031:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765010.488041:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765010/real 1775765010] req@ffff8b2382dad500 x1862022098753024/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765010.488049:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098753024/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765010.488055:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765010.488057:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765010.488059:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098753024:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765010.488063:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098753152:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765010.488079:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765010.488089:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765010/real 1775765010] req@ffff8b2382dae680 x1862022098753152/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765010.488102:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098753152/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765010.488110:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765010.488114:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765010.488118:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098753152:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765010.488124:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098753280:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765010.488143:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765010.488151:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765010/real 1775765010] req@ffff8b2382dac380 x1862022098753280/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765010.488160:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098753280/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765010.488166:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765010.488168:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765010.488170:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098753280:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765011.383760:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765013.431732:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765015.479616:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765015.607569:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765015.607576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765015.607580:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765015.607591:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2477 00000100:00080000:1.0:1775765015.607602:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765015.607635:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765015.607670:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098753408/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765015.607701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765015.607706:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765015.607708:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765015.607713:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2477 00000100:00080000:1.0:1775765015.607718:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765015.607723:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765015.607735:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765015.607743:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098753408:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775765015.607752:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098753536/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765015.607766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765015.607769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765015.607780:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765015.607785:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2477 00000100:00080000:1.0:1775765015.607789:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765015.607794:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765015.607801:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765015.607812:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098753664/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765015.607820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765015.607822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765015.607823:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765015.607826:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2477 00000100:00080000:1.0:1775765015.607829:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765015.607832:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765015.607835:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765015.607841:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098753792/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765015.607848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765015.607850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765015.607852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765015.607855:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2477 00000100:00080000:1.0:1775765015.607857:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765015.607860:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765015.607864:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765015.607873:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098753920/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765015.607886:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098753536:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765015.607934:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098753664:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765015.607952:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098753792:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765015.607967:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098753920:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765017.527560:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765018.680028:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765015/real 1775765018] req@ffff8b23864fdf80 x1862022098753920/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765018.680063:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098753920/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765018.680076:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765018.680081:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765018.680086:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098753920:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765018.680094:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765015/real 1775765018] req@ffff8b23864fc380 x1862022098753536/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765018.680106:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098753536/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765018.680114:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765018.680117:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765018.680120:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098753536:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765018.680126:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765015/real 1775765018] req@ffff8b23864fed80 x1862022098753664/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765018.680138:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098753664/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765018.680144:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765018.680148:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765018.680151:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098753664:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765018.680156:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765015/real 1775765018] req@ffff8b23864fc000 x1862022098753792/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765018.680173:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098753792/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765018.680181:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765018.680184:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765018.680187:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098753792:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765018.680193:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765015/real 1775765018] req@ffff8b23864ff800 x1862022098753408/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765018.680203:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098753408/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765018.680210:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765018.680213:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765018.680217:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098753408:192.168.202.128@tcp:250:kworker.0 00010000:00010000:2.0:1775765019.575605:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765020.727618:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765020.727629:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765020.727634:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765020.727645:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2482 00000100:00080000:2.0:1775765020.727657:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765020.727664:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765020.727708:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7800 x1862022098754048/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765020.727737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765020.727741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765020.727742:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765020.727745:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2482 00000100:00080000:2.0:1775765020.727747:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765020.727750:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765020.727759:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765020.727768:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022098754176/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765020.727777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765020.727780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765020.727782:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765020.727786:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2482 00000100:00080000:2.0:1775765020.727788:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765020.727792:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765020.727796:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765020.727806:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098754304/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765020.727814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765020.727817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765020.727819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765020.727829:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2482 00000100:00080000:2.0:1775765020.727832:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765020.727834:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765020.727839:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765020.727848:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098754432/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765020.727855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765020.727883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765020.727885:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765020.727890:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2482 00000100:00080000:2.0:1775765020.727898:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765020.727903:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765020.727907:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765020.727918:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098754560/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765021.623658:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765023.671605:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765025.719585:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765025.847565:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765025.847574:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765025.847578:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765025.847586:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2487 00000100:00080000:2.0:1775765025.847597:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765025.847604:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765025.847640:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098754688/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765025.847666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765025.847669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765025.847671:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765025.847673:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2487 00000100:00080000:2.0:1775765025.847677:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765025.847680:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765025.847688:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765025.847704:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098754816/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765025.847713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765025.847716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765025.847717:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765025.847722:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2487 00000100:00080000:2.0:1775765025.847724:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765025.847728:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765025.847735:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765025.847744:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098754944/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765025.847750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765025.847753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765025.847754:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765025.847757:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2487 00000100:00080000:2.0:1775765025.847759:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765025.847761:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765025.847765:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765025.847773:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098755072/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765025.847780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765025.847782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765025.847783:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765025.847786:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2487 00000100:00080000:2.0:1775765025.847788:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765025.847790:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765025.847793:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765025.847802:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098755200/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765027.767586:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765029.815787:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765030.967556:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765030.967564:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765030.967567:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765030.967575:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2492 00000100:00080000:1.0:1775765030.967586:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765030.967594:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765030.967632:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098755328/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765030.967665:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765030.967669:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765030.967671:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765030.967675:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2492 00000100:00080000:1.0:1775765030.967679:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765030.967683:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765030.967693:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765030.967715:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098755456/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765030.967736:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765030.967740:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765030.967742:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765030.967747:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2492 00000100:00080000:1.0:1775765030.967751:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765030.967755:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765030.967775:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765030.967788:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098755584/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765030.967797:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765030.967801:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765030.967805:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765030.967812:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2492 00000100:00080000:1.0:1775765030.967818:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765030.967823:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765030.967829:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765030.967844:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098755712/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765030.967853:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765030.967857:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765030.967859:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765030.967871:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2492 00000100:00080000:1.0:1775765030.967879:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765030.967885:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765030.967891:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765030.967905:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098755840/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765031.863556:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765033.911557:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765035.959560:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765036.087570:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765036.087578:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765036.087588:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765036.087596:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2497 00000100:00080000:2.0:1775765036.087605:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765036.087611:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765036.087644:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098755968/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765036.087669:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765036.087672:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765036.087673:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765036.087676:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2497 00000100:00080000:2.0:1775765036.087678:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765036.087680:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765036.087687:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765036.087695:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098756096/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765036.087702:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765036.087704:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765036.087705:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765036.087708:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2497 00000100:00080000:2.0:1775765036.087710:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765036.087712:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765036.087727:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765036.087734:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098756224/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765036.087740:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765036.087744:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765036.087745:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765036.087748:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2497 00000100:00080000:2.0:1775765036.087752:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765036.087755:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765036.087758:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765036.087765:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022098756352/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765036.087771:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765036.087774:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765036.087775:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765036.087777:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2497 00000100:00080000:2.0:1775765036.087779:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765036.087781:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765036.087783:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765036.087791:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098756480/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765038.007631:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765040.055605:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765041.207569:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765041.207583:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765041.207589:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765041.207604:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2502 00000100:00080000:2.0:1775765041.207618:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765041.207625:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765041.207682:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022098756608/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765041.207725:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765041.207730:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765041.207733:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765041.207738:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2502 00000100:00080000:2.0:1775765041.207742:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765041.207747:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765041.207759:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765041.207772:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5180 x1862022098756736/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765041.207784:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765041.207788:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765041.207793:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765041.207801:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2502 00000100:00080000:2.0:1775765041.207809:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765041.207815:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765041.207818:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765041.207828:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022098756864/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765041.207843:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765041.207847:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765041.207849:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765041.207852:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2502 00000100:00080000:2.0:1775765041.207857:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765041.207861:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765041.207865:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765041.207886:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098756992/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765041.207894:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765041.207897:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765041.207899:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765041.207905:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2502 00000100:00080000:2.0:1775765041.207908:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765041.207910:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765041.207916:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765041.207933:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022098757120/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765042.103651:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765044.151679:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765046.199942:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765046.327539:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765046.327547:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765046.327551:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765046.327559:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2507 00000100:00080000:2.0:1775765046.327570:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765046.327576:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765046.327610:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098757248/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765046.327655:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765046.327658:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765046.327660:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765046.327663:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2507 00000100:00080000:2.0:1775765046.327666:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765046.327669:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765046.327677:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765046.327687:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098757376/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765046.327697:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765046.327700:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765046.327703:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765046.327710:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2507 00000100:00080000:2.0:1775765046.327712:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765046.327714:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765046.327717:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765046.327726:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098757504/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765046.327732:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765046.327735:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765046.327736:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765046.327739:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2507 00000100:00080000:2.0:1775765046.327749:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765046.327752:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765046.327755:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765046.327764:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098757632/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765046.327771:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765046.327774:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765046.327775:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765046.327779:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2507 00000100:00080000:2.0:1775765046.327781:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765046.327783:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765046.327786:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765046.327794:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098757760/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765046.327994:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098757632:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765046.328013:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765046.328022:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765046/real 1775765046] req@ffff8b23b9ea4000 x1862022098757632/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765046.328033:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022098757632/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765046.328040:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765046.328042:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765046.328045:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098757632:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765046.328050:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098757760:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765046.328087:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765046.328098:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765046/real 1775765046] req@ffff8b23b9ea5500 x1862022098757760/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765046.328113:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022098757760/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765046.328124:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765046.328128:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765046.328133:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098757760:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775765047.223566:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765048.247535:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765049.271579:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765050.295533:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765051.319620:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765051.447529:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765051.447545:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765051.447550:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765051.447561:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2512 00000100:00080000:2.0:1775765051.447590:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765051.447598:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765051.447639:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098757888/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765051.447682:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765051.447687:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765051.447697:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765051.447702:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2512 00000100:00080000:2.0:1775765051.447707:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765051.447711:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765051.447719:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098757888:192.168.202.128@tcp:250:kworker.0 00000008:00100000:2.0:1775765051.447721:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765051.447739:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098758016/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765051.447751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765051.447754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765051.447756:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765051.447759:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2512 00000100:00080000:2.0:1775765051.447763:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765051.447766:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765051.447770:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765051.447780:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098758144/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765051.447788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765051.447791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765051.447792:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765051.447796:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2512 00000100:00080000:2.0:1775765051.447800:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765051.447803:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765051.447807:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765051.447816:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098758272/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765051.447824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765051.447826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765051.447828:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765051.447832:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2512 00000100:00080000:2.0:1775765051.447834:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765051.447838:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765051.447842:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765051.447850:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098758400/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765051.447856:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098758016:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765051.447901:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098758144:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765051.447953:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098758272:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765051.447995:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098758400:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765052.343545:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765053.367629:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765054.391676:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775765054.519941:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775765054.520012:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775765054.520044:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775765054.520120:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775765054.520140:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775765054.520156:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775765054.520169:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:3.0:1775765054.520176:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765051/real 1775765054] req@ffff8b23b9ea4700 x1862022098758016/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765054.520240:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022098758016/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765054.520262:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765054.520271:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765054.520278:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098758016:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765054.520293:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765051/real 1775765054] req@ffff8b23b9ea4a80 x1862022098758144/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765054.520317:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022098758144/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765054.520330:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765054.520336:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765054.520342:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098758144:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765054.520354:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765051/real 1775765054] req@ffff8b23b9ea7100 x1862022098758272/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765054.520389:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022098758272/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765054.520402:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765054.520409:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765054.520415:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098758272:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765054.520427:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765051/real 1775765054] req@ffff8b23b9ea4380 x1862022098757888/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765054.520448:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022098757888/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765054.520460:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765054.520466:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765054.520517:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098757888:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765054.520589:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765051/real 1775765054] req@ffff8b23b9ea4e00 x1862022098758400/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765054.520611:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022098758400/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765054.520623:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765054.520630:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765054.520635:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098758400:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775765055.415608:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765056.439629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765056.567630:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765056.567641:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765056.567646:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765056.567657:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2517 00000100:00080000:3.0:1775765056.567668:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765056.567675:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765056.567712:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098758528/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765056.567745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765056.567748:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765056.567749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765056.567752:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2517 00000100:00080000:3.0:1775765056.567758:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765056.567761:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765056.567770:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765056.567785:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098758656/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765056.567794:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098758528:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765056.567812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765056.567816:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765056.567819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765056.567828:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2517 00000100:00080000:3.0:1775765056.567834:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765056.567840:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765056.567846:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765056.567870:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098758784/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765056.567882:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765056.567886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765056.567888:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765056.567892:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2517 00000100:00080000:3.0:1775765056.567898:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765056.567903:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775765056.567908:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765056.567915:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765056.567932:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098758912/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765056.567943:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765056.567947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765056.567950:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765056.567954:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2517 00000100:00080000:3.0:1775765056.567958:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765056.567962:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765056.567967:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765056.567982:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098759040/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765056.567990:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765056/real 1775765056] req@ffff8b23864f2d80 x1862022098758528/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765056.568028:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098758528/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765056.568052:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765056.568064:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765056.568070:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098758528:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765056.568108:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098758656:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765056.568139:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765056.568154:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765056/real 1775765056] req@ffff8b23864f3b80 x1862022098758656/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765056.568168:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098758656/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765056.568176:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765056.568180:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765056.568183:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098758656:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765056.568191:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098758784:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765056.568213:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765056.568225:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765056/real 1775765056] req@ffff8b23864f3480 x1862022098758784/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765056.568238:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098758784/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765056.568246:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765056.568249:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765056.568252:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098758784:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765056.568260:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098758912:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765056.568285:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765056.568298:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765056/real 1775765056] req@ffff8b23864f0700 x1862022098758912/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765056.568311:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098758912/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765056.568320:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765056.568323:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765056.568327:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098758912:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765056.568334:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098759040:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765056.568370:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765056.568388:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765056/real 1775765056] req@ffff8b23864f1180 x1862022098759040/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765056.568402:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098759040/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765056.568411:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765056.568417:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765056.568421:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098759040:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775765057.463626:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765058.487568:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765059.511655:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765060.535629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765061.559664:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765061.687599:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765061.687606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765061.687610:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765061.687619:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2523 00000100:00080000:1.0:1775765061.687628:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765061.687633:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765061.687665:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098759168/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765061.687696:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765061.687700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765061.687702:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765061.687706:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2523 00000100:00080000:1.0:1775765061.687710:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765061.687714:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765061.687724:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765061.687739:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098759296/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765061.687752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765061.687755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765061.687756:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765061.687760:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2523 00000100:00080000:1.0:1775765061.687763:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765061.687766:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765061.687771:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765061.687781:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098759168:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775765061.687783:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098759424/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765061.687817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765061.687819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765061.687820:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765061.687824:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2523 00000100:00080000:1.0:1775765061.687827:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765061.687830:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765061.687834:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765061.687842:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098759552/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765061.687858:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765061.687861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765061.687863:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765061.687865:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2523 00000100:00080000:1.0:1775765061.687872:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765061.687874:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765061.687878:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765061.687886:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098759680/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765061.687902:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765061.687952:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765061/real 1775765061] req@ffff8b23864fca80 x1862022098759168/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765061.687968:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098759168/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765061.687977:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765061.687981:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765061.687985:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098759168:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765061.687990:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098759296:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765061.688006:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765061.688014:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765061/real 1775765061] req@ffff8b23864fe680 x1862022098759296/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765061.688022:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098759296/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765061.688027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765061.688029:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765061.688031:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098759296:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765061.688057:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098759424:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765061.688068:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765061.688074:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765061/real 1775765061] req@ffff8b23864fce00 x1862022098759424/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765061.688087:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098759424/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765061.688092:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765061.688093:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765061.688095:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098759424:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765061.688099:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098759552:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765061.688110:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765061.688116:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765061/real 1775765061] req@ffff8b23864fd180 x1862022098759552/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765061.688123:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098759552/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765061.688128:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765061.688130:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765061.688131:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098759552:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765061.688135:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098759680:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765061.688144:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765061.688149:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765061/real 1775765061] req@ffff8b23864fea00 x1862022098759680/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765061.688156:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098759680/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765061.688165:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765061.688167:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765061.688170:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098759680:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765062.583622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765063.607640:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765064.631633:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765065.655649:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765066.679565:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765066.807542:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765066.807553:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765066.807558:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765066.807570:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2528 00000100:00080000:1.0:1775765066.807583:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765066.807590:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765066.807642:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098759808/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765066.807665:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765066.807668:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765066.807669:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765066.807672:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2528 00000100:00080000:1.0:1775765066.807675:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765066.807677:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765066.807688:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765066.807697:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098759936/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765066.807707:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765066.807708:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098759808:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775765066.807713:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765066.807717:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765066.807721:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2528 00000100:00080000:1.0:1775765066.807723:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765066.807725:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765066.807728:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765066.807737:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098760064/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765066.807744:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765066.807749:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765066.807751:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765066.807756:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2528 00000100:00080000:1.0:1775765066.807760:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765066.807764:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765066.807770:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765066.807785:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098760192/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765066.807787:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775765066.807800:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765066.807804:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765066.807807:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765066.807812:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2528 00000100:00080000:1.0:1775765066.807817:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765066.807820:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765066.807825:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765066.807839:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098760320/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765066.807839:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765066/real 1775765066] req@ffff8b23864fdc00 x1862022098759808/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765066.807874:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098759808/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765066.807883:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765066.807887:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765066.807892:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098759808:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765066.807915:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098759936:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765066.807931:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765066.807940:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765066/real 1775765066] req@ffff8b23864fd500 x1862022098759936/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765066.807948:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098759936/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765066.807958:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765066.807961:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765066.807963:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098759936:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765066.807967:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098760064:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765066.807977:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765066.807983:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765066/real 1775765066] req@ffff8b23864ff100 x1862022098760064/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765066.807990:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098760064/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765066.807996:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765066.807998:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765066.807999:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098760064:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765066.808003:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098760192:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765066.808014:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765066.808019:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765066/real 1775765066] req@ffff8b23864fe300 x1862022098760192/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765066.808027:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098760192/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765066.808031:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765066.808033:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765066.808036:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098760192:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765066.808040:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098760320:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765066.808050:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765066.808055:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765066/real 1775765066] req@ffff8b23864ffb80 x1862022098760320/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765066.808062:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098760320/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765066.808067:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765066.808069:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765066.808071:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098760320:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775765067.703654:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765068.727645:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765069.751632:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765070.775601:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765071.799625:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765071.927585:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765071.927593:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765071.927597:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765071.927604:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2533 00000100:00080000:3.0:1775765071.927614:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765071.927619:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765071.927653:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098760448/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765071.927675:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765071.927678:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765071.927679:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765071.927682:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2533 00000100:00080000:3.0:1775765071.927684:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765071.927687:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765071.927695:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765071.927705:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098760576/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765071.927713:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765071.927716:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765071.927716:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765071.927719:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2533 00000100:00080000:3.0:1775765071.927723:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765071.927726:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765071.927729:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765071.927737:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098760704/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765071.927744:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098760448:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765071.927749:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765071.927755:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765071.927756:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765071.927759:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2533 00000100:00080000:3.0:1775765071.927761:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765071.927763:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765071.927767:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765071.927777:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098760832/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765071.927783:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765071.927785:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765071.927786:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765071.927789:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2533 00000100:00080000:3.0:1775765071.927791:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765071.927794:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765071.927797:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765071.927813:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098760960/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765071.927833:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765071.927881:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765071/real 1775765071] req@ffff8b23864f3100 x1862022098760448/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765071.927897:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098760448/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765071.927906:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765071.927910:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765071.927913:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098760448:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765071.927920:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098760576:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765071.927936:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765071.927945:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765071/real 1775765071] req@ffff8b23864f0000 x1862022098760576/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765071.927953:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098760576/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765071.927958:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765071.927960:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765071.927962:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098760576:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765071.927986:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098760704:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765071.927998:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765071.928004:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765071/real 1775765071] req@ffff8b23864f2a00 x1862022098760704/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765071.928011:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098760704/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765071.928017:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765071.928018:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765071.928020:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098760704:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765071.928025:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098760832:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765071.928036:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765071.928042:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765071/real 1775765071] req@ffff8b23864f2680 x1862022098760832/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765071.928054:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098760832/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765071.928060:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765071.928061:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765071.928063:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098760832:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765071.928066:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098760960:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765071.928076:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765071.928082:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765071/real 1775765071] req@ffff8b23864f1c00 x1862022098760960/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765071.928088:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098760960/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765071.928093:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765071.928094:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765071.928096:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098760960:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765072.823560:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765073.847718:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765074.871654:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765075.895676:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765076.919593:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765077.047564:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765077.047572:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765077.047576:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765077.047584:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2538 00000100:00080000:3.0:1775765077.047595:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765077.047602:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765077.047647:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098761088/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765077.047672:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765077.047675:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765077.047677:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765077.047679:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2538 00000100:00080000:3.0:1775765077.047681:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765077.047684:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765077.047691:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765077.047699:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098761216/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765077.047707:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765077.047709:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765077.047710:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765077.047713:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2538 00000100:00080000:3.0:1775765077.047716:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765077.047720:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765077.047723:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765077.047732:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098761344/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765077.047740:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098761088:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765077.047742:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765077.047749:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765077.047750:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765077.047753:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2538 00000100:00080000:3.0:1775765077.047755:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765077.047766:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765077.047769:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765077.047777:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098761472/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765077.047783:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765077.047785:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765077.047786:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765077.047789:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2538 00000100:00080000:3.0:1775765077.047800:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765077.047803:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765077.047806:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765077.047815:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765077.047818:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098761600/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765077.047872:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765077/real 1775765077] req@ffff8b23864f0a80 x1862022098761088/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765077.047891:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098761088/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765077.047900:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765077.047905:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765077.047909:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098761088:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765077.047918:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098761216:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765077.047943:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765077.047957:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765077/real 1775765077] req@ffff8b23864f0380 x1862022098761216/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765077.047969:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098761216/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765077.047975:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765077.047977:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765077.047979:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098761216:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765077.047997:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098761344:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765077.048012:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765077.048022:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765077/real 1775765077] req@ffff8b23864f1f80 x1862022098761344/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765077.048034:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098761344/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765077.048049:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765077.048052:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765077.048055:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098761344:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765077.048062:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098761472:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765077.048084:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765077.048097:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765077/real 1775765077] req@ffff8b23864f1c00 x1862022098761472/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765077.048127:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098761472/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765077.048134:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765077.048137:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765077.048139:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098761472:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765077.048144:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098761600:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765077.048160:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765077.048167:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765077/real 1775765077] req@ffff8b23864f2680 x1862022098761600/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765077.048184:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098761600/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765077.048189:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765077.048191:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765077.048193:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098761600:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775765077.943630:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765078.967544:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765079.991652:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765081.015548:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765082.039575:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765082.167541:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765082.167552:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765082.167557:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765082.167567:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2543 00000100:00080000:3.0:1775765082.167579:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765082.167598:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765082.167635:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098761728/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765082.167670:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765082.167673:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765082.167674:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765082.167678:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2543 00000100:00080000:3.0:1775765082.167680:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765082.167683:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765082.167691:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765082.167702:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098761856/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765082.167711:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765082.167713:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765082.167715:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765082.167731:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098761728:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765082.167732:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2543 00000100:00080000:3.0:1775765082.167736:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765082.167742:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765082.167745:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765082.167753:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098761984/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765082.167759:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765082.167761:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765082.167762:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765082.167765:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2543 00000100:00080000:3.0:1775765082.167767:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765082.167769:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765082.167772:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765082.167779:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098762112/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765082.167784:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765082.167787:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765082.167788:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765082.167791:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2543 00000100:00080000:3.0:1775765082.167794:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765082.167795:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765082.167798:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765082.167804:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765082.167807:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098762240/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765082.167852:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765082/real 1775765082] req@ffff8b23864f2a00 x1862022098761728/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765082.167869:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098761728/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765082.167878:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765082.167893:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765082.167896:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098761728:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765082.167916:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098761856:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765082.167933:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765082.167941:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765082/real 1775765082] req@ffff8b23864f0000 x1862022098761856/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765082.167949:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098761856/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765082.167954:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765082.167957:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765082.167959:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098761856:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765082.167962:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098761984:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765082.167973:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765082.167979:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765082/real 1775765082] req@ffff8b23864f3100 x1862022098761984/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765082.167987:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098761984/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765082.167991:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765082.167993:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765082.167995:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098761984:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765082.167999:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098762112:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765082.168009:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765082.168014:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765082/real 1775765082] req@ffff8b23864f1180 x1862022098762112/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765082.168021:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098762112/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765082.168026:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765082.168027:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765082.168029:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098762112:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765082.168032:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098762240:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765082.168050:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765082.168055:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765082/real 1775765082] req@ffff8b23864f0700 x1862022098762240/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765082.168067:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098762240/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765082.168072:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765082.168075:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765082.168078:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098762240:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765083.063548:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765084.087694:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765085.111653:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765086.135570:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765087.159543:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765087.287544:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765087.287554:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765087.287558:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765087.287568:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2548 00000100:00080000:2.0:1775765087.287581:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765087.287588:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765087.287628:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098762368/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765087.287656:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765087.287659:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765087.287661:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765087.287664:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2548 00000100:00080000:2.0:1775765087.287668:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765087.287671:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765087.287679:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765087.287686:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098762368:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765087.287692:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022098762496/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765087.287701:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765087.287704:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765087.287705:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765087.287708:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2548 00000100:00080000:2.0:1775765087.287710:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765087.287713:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765087.287716:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765087.287723:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7800 x1862022098762624/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765087.287730:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765087.287733:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765087.287734:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765087.287736:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2548 00000100:00080000:2.0:1775765087.287739:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765087.287743:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775765087.287744:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:2.0:1775765087.287749:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765087.287758:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098762752/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765087.287765:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765087.287767:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765087.287769:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765087.287772:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2548 00000100:00080000:2.0:1775765087.287774:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765087.287777:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765087.287779:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765087.287788:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098762880/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765087.287793:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765087/real 1775765087] req@ffff8b23b9ea7480 x1862022098762368/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765087.287811:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022098762368/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765087.287820:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765087.287824:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765087.287828:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098762368:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765087.287846:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098762496:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765087.287868:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765087.287882:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765087/real 1775765087] req@ffff8b23b9ea6300 x1862022098762496/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765087.287896:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022098762496/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765087.287905:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765087.287908:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765087.287912:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098762496:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765087.287919:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098762624:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765087.287978:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765087.287987:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765087/real 1775765087] req@ffff8b23b9ea7800 x1862022098762624/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765087.287995:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7800 x1862022098762624/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765087.288000:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765087.288002:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765087.288004:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098762624:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765087.288008:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098762752:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765087.288021:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765087.288027:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765087/real 1775765087] req@ffff8b23b9ea4e00 x1862022098762752/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765087.288034:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022098762752/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765087.288039:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765087.288040:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765087.288042:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098762752:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765087.288046:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098762880:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765087.288056:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765087.288061:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765087/real 1775765087] req@ffff8b23b9ea4380 x1862022098762880/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765087.288072:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022098762880/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765087.288077:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765087.288079:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765087.288081:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098762880:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775765088.183562:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765089.207583:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765090.231565:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765091.255558:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765092.279591:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765092.407541:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765092.407550:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765092.407554:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765092.407564:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2553 00000100:00080000:3.0:1775765092.407575:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765092.407582:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765092.407619:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098763008/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765092.407648:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765092.407652:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765092.407654:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765092.407658:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2553 00000100:00080000:3.0:1775765092.407663:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765092.407667:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765092.407677:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765092.407680:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098763008:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765092.407693:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098763136/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765092.407707:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765092.407711:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765092.407713:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765092.407717:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2553 00000100:00080000:3.0:1775765092.407723:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765092.407727:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765092.407732:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765092.407738:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765092.407750:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098763264/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765092.407760:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765092.407764:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765092.407766:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765092.407770:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2553 00000100:00080000:3.0:1775765092.407774:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765092.407777:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765092.407783:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765092.407784:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765092/real 1775765092] req@ffff8b23864f3480 x1862022098763008/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765092.407798:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098763392/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765092.407808:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098763008/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765092.407810:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765092.407818:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765092.407823:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765092.407832:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2553 00000100:00080000:2.0:1775765092.407833:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765092.407837:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765092.407839:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00080000:3.0:1775765092.407845:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765092.407846:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098763008:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775765092.407855:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765092.407869:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098763520/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765092.407873:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098763136:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765092.407901:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765092.407921:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765092/real 1775765092] req@ffff8b23864f3b80 x1862022098763136/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765092.407929:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098763136/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765092.407939:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765092.407942:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765092.407944:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098763136:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765092.407948:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098763264:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765092.407960:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765092.407967:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765092/real 1775765092] req@ffff8b23864f2d80 x1862022098763264/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765092.407974:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098763264/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765092.407979:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765092.407981:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765092.407983:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098763264:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765092.407986:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098763392:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765092.408003:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765092.408013:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765092/real 1775765092] req@ffff8b23864f0700 x1862022098763392/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765092.408025:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098763392/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765092.408034:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765092.408037:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765092.408041:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098763392:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765092.408057:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098763520:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765092.408079:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765092.408103:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765092/real 1775765092] req@ffff8b23864f1180 x1862022098763520/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765092.408115:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098763520/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765092.408124:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765092.408127:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765092.408131:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098763520:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765093.303544:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765094.327600:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765095.351544:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765097.399675:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765097.527576:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765097.527594:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765097.527616:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765097.527629:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2558 00000100:00080000:3.0:1775765097.527642:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765097.527652:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765097.527713:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098763648/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765097.527747:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765097.527751:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765097.527753:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765097.527756:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2558 00000100:00080000:3.0:1775765097.527761:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765097.527765:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765097.527775:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765097.527792:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098763776/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765097.527813:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098763648:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765097.527813:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765097.527825:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765097.527828:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765097.527833:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2558 00000100:00080000:3.0:1775765097.527836:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765097.527840:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765097.527844:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765097.527857:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098763904/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765097.527867:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765097.527870:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765097.527872:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765097.527876:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2558 00000100:00080000:3.0:1775765097.527881:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765097.527884:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765097.527889:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765097.527894:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765097.527905:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098764032/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765097.527915:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765097.527918:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765097.527920:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765097.527923:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2558 00000100:00080000:3.0:1775765097.527928:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765097.527932:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765097.527937:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765097.527941:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765097/real 1775765097] req@ffff8b23864f3100 x1862022098763648/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765097.527952:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098764160/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765097.527971:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098763648/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765097.527984:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765097.527989:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765097.527994:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098763648:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765097.528019:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098763776:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765097.528063:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765097.528078:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765097/real 1775765097] req@ffff8b23864f0000 x1862022098763776/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765097.528108:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098763776/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765097.528116:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765097.528120:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765097.528123:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098763776:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765097.528128:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098763904:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765097.528165:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765097.528180:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765097/real 1775765097] req@ffff8b23864f2a00 x1862022098763904/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765097.528192:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098763904/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765097.528200:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765097.528203:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765097.528206:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098763904:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765097.528212:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098764032:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765097.528232:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765097.528264:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765097/real 1775765097] req@ffff8b23864f2680 x1862022098764032/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765097.528287:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098764032/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765097.528301:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765097.528306:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765097.528312:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098764032:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765097.528341:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098764160:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765097.528378:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765097.528398:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765097/real 1775765097] req@ffff8b23864f1c00 x1862022098764160/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765097.528422:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098764160/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765097.528456:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765097.528460:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765097.528464:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098764160:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765099.447556:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765101.495568:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765102.647574:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765102.647583:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765102.647587:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765102.647596:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2564 00000100:00080000:1.0:1775765102.647606:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765102.647612:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765102.647646:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098764288/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765102.647671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765102.647674:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765102.647675:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765102.647678:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2564 00000100:00080000:1.0:1775765102.647681:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765102.647683:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765102.647691:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765102.647705:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098764416/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765102.647714:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098764288:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775765102.647730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765102.647732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765102.647734:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765102.647738:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2564 00000100:00080000:1.0:1775765102.647741:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765102.647743:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765102.647747:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765102.647757:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098764544/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765102.647763:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765102.647766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765102.647768:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765102.647771:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2564 00000100:00080000:1.0:1775765102.647774:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765102.647776:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:2.0:1775765102.647781:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:1.0:1775765102.647786:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765102.647797:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098764672/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765102.647804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765102.647807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765102.647808:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765102.647811:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2564 00000100:00080000:1.0:1775765102.647814:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765102.647817:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765102.647820:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765102.647830:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765102/real 1775765102] req@ffff8b23864ffb80 x1862022098764288/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765102.647832:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098764800/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765102.647877:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098764288/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765102.647887:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765102.647891:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765102.647894:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098764288:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765102.647913:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098764416:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765102.647929:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765102.647938:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765102/real 1775765102] req@ffff8b23864fe300 x1862022098764416/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765102.647965:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098764416/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765102.647975:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765102.647980:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765102.647985:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098764416:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765102.647993:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098764544:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765102.648020:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765102.648034:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765102/real 1775765102] req@ffff8b23864ff100 x1862022098764544/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765102.648060:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098764544/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765102.648072:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765102.648075:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765102.648078:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098764544:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765102.648135:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098764672:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765102.648161:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765102.648174:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765102/real 1775765102] req@ffff8b23864fd500 x1862022098764672/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765102.648185:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098764672/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765102.648191:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765102.648194:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765102.648196:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098764672:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765102.648201:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098764800:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765102.648215:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765102.648222:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765102/real 1775765102] req@ffff8b23864fdc00 x1862022098764800/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765102.648229:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098764800/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765102.648234:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765102.648236:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765102.648238:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098764800:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765103.543652:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765105.591630:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765107.639565:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765107.767557:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765107.767565:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765107.767569:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765107.767577:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2569 00000100:00080000:2.0:1775765107.767587:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765107.767628:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765107.767661:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098764928/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765107.767684:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765107.767687:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765107.767688:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765107.767691:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2569 00000100:00080000:2.0:1775765107.767693:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765107.767696:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765107.767704:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765107.767713:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098765056/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765107.767721:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098764928:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775765107.767724:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765107.767730:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765107.767731:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765107.767735:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2569 00000100:00080000:2.0:1775765107.767737:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765107.767740:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765107.767743:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765107.767750:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098765184/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765107.767758:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765107.767760:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765107.767761:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765107.767764:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2569 00000100:00080000:2.0:1775765107.767767:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765107.767769:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765107.767772:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765107.767782:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098765312/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765107.767782:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775765107.767790:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765107.767793:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765107.767795:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765107.767798:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2569 00000100:00080000:2.0:1775765107.767800:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765107.767803:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765107.767805:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765107.767811:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098765440/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765107.767829:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765107/real 1775765107] req@ffff8b23b9ea7100 x1862022098764928/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765107.767844:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022098764928/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765107.767852:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765107.767855:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765107.767858:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098764928:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765107.767877:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098765056:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765107.767892:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765107.767900:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765107/real 1775765107] req@ffff8b23b9ea4a80 x1862022098765056/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765107.767908:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022098765056/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765107.767913:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765107.767916:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765107.767918:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098765056:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765107.767922:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098765184:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765107.767934:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765107.767939:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765107/real 1775765107] req@ffff8b23b9ea4700 x1862022098765184/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765107.767946:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022098765184/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765107.767951:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765107.767952:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765107.767954:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098765184:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765107.767958:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098765312:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765107.767968:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765107.767972:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765107/real 1775765107] req@ffff8b23b9ea5500 x1862022098765312/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765107.767980:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022098765312/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765107.767984:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765107.767986:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765107.767988:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098765312:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765107.767991:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098765440:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765107.768000:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765107.768005:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765107/real 1775765107] req@ffff8b23b9ea4000 x1862022098765440/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765107.768012:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022098765440/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765107.768017:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765107.768019:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765107.768021:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098765440:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765109.687552:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765111.735533:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765112.887533:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765112.887541:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765112.887545:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765112.887553:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2574 00000100:00080000:3.0:1775765112.887563:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765112.887568:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765112.887600:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098765568/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765112.887624:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765112.887626:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765112.887628:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765112.887631:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2574 00000100:00080000:3.0:1775765112.887633:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765112.887636:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765112.887644:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765112.887652:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098765696/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765112.887657:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098765568:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765112.887664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765112.887667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765112.887668:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765112.887672:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2574 00000100:00080000:3.0:1775765112.887674:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765112.887677:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765112.887680:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765112.887687:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098765824/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765112.887693:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765112.887696:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765112.887697:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765112.887700:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2574 00000100:00080000:3.0:1775765112.887711:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765112.887714:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765112.887717:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765112.887725:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765112.887730:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098765952/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765112.887740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765112.887743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765112.887745:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765112.887748:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2574 00000100:00080000:3.0:1775765112.887753:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765112.887757:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765112.887764:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765112.887798:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098766080/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765112.887808:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765112/real 1775765112] req@ffff8b23864f1f80 x1862022098765568/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765112.887842:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098765568/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765112.887851:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765112.887854:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765112.887857:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098765568:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765112.887876:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098765696:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765112.887894:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765112.887903:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765112/real 1775765112] req@ffff8b23864f0380 x1862022098765696/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765112.887912:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098765696/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765112.887917:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765112.887919:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765112.887922:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098765696:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765112.887925:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098765824:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765112.887936:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765112.887955:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765112/real 1775765112] req@ffff8b23864f0a80 x1862022098765824/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765112.887964:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098765824/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765112.887974:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765112.887976:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765112.887994:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098765824:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765112.887998:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098765952:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765112.888040:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765112.888054:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765112/real 1775765112] req@ffff8b23864f1c00 x1862022098765952/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765112.888062:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098765952/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765112.888068:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765112.888070:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765112.888072:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098765952:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765112.888076:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098766080:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765112.888087:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765112.888093:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765112/real 1775765112] req@ffff8b23864f2680 x1862022098766080/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765112.888100:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098766080/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765112.888105:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765112.888106:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765112.888109:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098766080:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765113.783586:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765115.831538:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765117.879545:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765118.007595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765118.007607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765118.007614:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765118.007625:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2579 00000100:00080000:3.0:1775765118.007641:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765118.007652:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765118.007723:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098766208/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765118.007770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765118.007777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765118.007780:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765118.007787:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2579 00000100:00080000:3.0:1775765118.007795:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765118.007800:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765118.007812:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765118.007829:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098766336/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765118.007843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765118.007850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765118.007866:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765118.007871:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2579 00000100:00080000:3.0:1775765118.007875:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765118.007884:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765118.007887:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098766208:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775765118.007890:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765118.007908:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098766464/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765118.007919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765118.007923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765118.007925:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765118.007928:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2579 00000100:00080000:3.0:1775765118.007932:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765118.007935:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765118.007974:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765118.007992:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098766592/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765118.008007:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765118.008012:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765118.008015:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765118.008020:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2579 00000100:00080000:3.0:1775765118.008027:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765118.008033:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765118.008042:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765118.008060:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098766720/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765118.008074:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098766336:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765118.008123:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098766464:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765118.008168:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098766592:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765118.008191:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098766720:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765119.927599:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765121.080058:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765118/real 1775765121] req@ffff8b23864f3100 x1862022098766464/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765121.080105:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098766464/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765121.080126:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765121.080135:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765121.080144:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098766464:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765121.080158:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765118/real 1775765121] req@ffff8b23864f1180 x1862022098766592/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765121.080181:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098766592/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765121.080207:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765121.080213:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765121.080220:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098766592:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765121.080232:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765118/real 1775765121] req@ffff8b23864f0700 x1862022098766720/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765121.080255:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098766720/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765121.080284:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765121.080291:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765121.080297:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098766720:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765121.080310:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765118/real 1775765121] req@ffff8b23864f2a00 x1862022098766208/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765121.080333:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098766208/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765121.080377:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765121.080385:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765121.080391:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098766208:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765121.080403:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765118/real 1775765121] req@ffff8b23864f0000 x1862022098766336/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765121.080424:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098766336/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765121.080438:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765121.080444:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765121.080489:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098766336:192.168.202.128@tcp:38:kworker.0 00010000:00010000:2.0:1775765121.975555:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765123.127658:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765123.127673:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765123.127681:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765123.127711:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2584 00000100:00080000:2.0:1775765123.127724:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765123.127732:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765123.127778:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098766848/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765123.127818:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765123.127825:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765123.127828:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765123.127835:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2584 00000100:00080000:2.0:1775765123.127842:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765123.127848:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765123.127864:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765123.127892:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098766976/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765123.127930:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098766848:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775765123.127965:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765123.127972:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765123.127976:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765123.127984:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2584 00000100:00080000:2.0:1775765123.127993:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765123.128001:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765123.128010:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765123.128034:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098767104/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765123.128042:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775765123.128061:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765123.128068:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765123.128072:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765123.128081:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2584 00000100:00080000:2.0:1775765123.128089:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765123.128095:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765123.128104:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765123.128127:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022098767232/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765123.128135:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765123/real 1775765123] req@ffff8b23b9ea6a00 x1862022098766848/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765123.128168:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765123.128176:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765123.128180:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775765123.128198:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022098766848/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765123.128201:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2584 00000100:00080000:2.0:1775765123.128216:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765123.128230:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:3.0:1775765123.128232:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1775765123.128246:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775765123.128248:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765123.128261:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098766848:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765123.128280:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098767360/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765123.128313:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098766976:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765123.128364:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765123.128388:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765123/real 1775765123] req@ffff8b23b9ea6680 x1862022098766976/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765123.128413:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022098766976/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765123.128430:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765123.128436:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765123.128443:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098766976:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765123.128510:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098767104:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765123.128548:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765123.128566:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765123/real 1775765123] req@ffff8b23b9ea5f80 x1862022098767104/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765123.128585:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022098767104/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765123.128597:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765123.128603:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765123.128608:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098767104:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765123.128618:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098767232:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765123.128650:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765123.128666:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765123/real 1775765123] req@ffff8b23b9ea7b80 x1862022098767232/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765123.128684:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022098767232/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765123.128696:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765123.128701:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765123.128706:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098767232:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765123.128716:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098767360:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765123.128979:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765123.129001:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765123/real 1775765123] req@ffff8b23b9ea5c00 x1862022098767360/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765123.129030:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022098767360/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765123.129043:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765123.129048:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765123.129053:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098767360:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765124.023560:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765126.071679:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765128.119568:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765128.247532:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765128.247546:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765128.247550:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765128.247558:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2589 00000100:00080000:2.0:1775765128.247568:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765128.247573:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765128.247605:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022098767488/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765128.247627:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765128.247630:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765128.247632:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765128.247634:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2589 00000100:00080000:2.0:1775765128.247636:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765128.247639:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765128.247646:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765128.247653:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5180 x1862022098767616/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765128.247660:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765128.247663:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765128.247664:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765128.247667:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2589 00000100:00080000:2.0:1775765128.247669:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765128.247671:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765128.247673:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765128.247679:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022098767744/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765128.247685:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765128.247687:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765128.247688:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765128.247691:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2589 00000100:00080000:2.0:1775765128.247693:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765128.247696:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765128.247698:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765128.247706:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098767872/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765128.247711:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098767488:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775765128.247716:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765128.247719:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765128.247720:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765128.247723:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2589 00000100:00080000:2.0:1775765128.247725:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765128.247727:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765128.247732:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765128.247740:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022098768000/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765128.247771:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765128.247821:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765128/real 1775765128] req@ffff8b23b9ea6d80 x1862022098767488/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765128.247835:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022098767488/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765128.247844:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765128.247847:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765128.247850:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098767488:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765128.247855:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098767616:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765128.247892:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765128.247912:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765128/real 1775765128] req@ffff8b23b9ea5180 x1862022098767616/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765128.247924:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5180 x1862022098767616/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765128.247939:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765128.247941:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765128.247943:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098767616:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765128.247948:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098767744:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765128.247962:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765128.247969:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765128/real 1775765128] req@ffff8b23b9ea5880 x1862022098767744/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765128.247976:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022098767744/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765128.247981:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765128.247983:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765128.247985:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098767744:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765128.248004:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098767872:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765128.248015:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765128.248020:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765128/real 1775765128] req@ffff8b23b9ea5c00 x1862022098767872/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765128.248027:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022098767872/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765128.248032:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765128.248034:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765128.248036:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098767872:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765128.248039:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098768000:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765128.248048:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765128.248053:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765128/real 1775765128] req@ffff8b23b9ea7b80 x1862022098768000/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765128.248059:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022098768000/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765128.248063:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765128.248065:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765128.248067:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098768000:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765130.167567:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765132.215614:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765133.367643:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765133.367655:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765133.367661:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765133.367673:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2594 00000100:00080000:3.0:1775765133.367687:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765133.367694:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765133.367747:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098768128/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765133.367779:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765133.367782:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765133.367784:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765133.367787:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2594 00000100:00080000:3.0:1775765133.367790:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765133.367794:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765133.367806:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765133.367843:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098768256/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765133.367846:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098768128:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765133.367865:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765133.367869:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765133.367871:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765133.367876:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2594 00000100:00080000:3.0:1775765133.367881:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765133.367887:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765133.367893:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765133.367908:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098768384/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765133.367917:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765133.367920:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765133.367921:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765133.367924:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2594 00000100:00080000:3.0:1775765133.367928:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765133.367930:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765133.367935:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765133.367945:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098768512/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765133.367954:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775765133.367955:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765133.367960:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765133.367962:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765133.367965:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2594 00000100:00080000:3.0:1775765133.367968:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765133.367971:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765133.367974:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765133.367981:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098768640/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765133.368032:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765133/real 1775765133] req@ffff8b23864f2d80 x1862022098768128/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765133.368057:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098768128/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765133.368071:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765133.368076:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765133.368081:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098768128:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765133.368112:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098768256:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765133.368139:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765133.368153:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765133/real 1775765133] req@ffff8b23864f3b80 x1862022098768256/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765133.368167:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098768256/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765133.368175:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765133.368178:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765133.368183:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098768256:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765133.368190:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098768384:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765133.368213:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765133.368226:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765133/real 1775765133] req@ffff8b23864f3480 x1862022098768384/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765133.368249:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098768384/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765133.368258:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765133.368262:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765133.368267:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098768384:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765133.368275:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098768512:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765133.368297:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765133.368309:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765133/real 1775765133] req@ffff8b23864f0000 x1862022098768512/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765133.368322:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098768512/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765133.368330:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765133.368334:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765133.368337:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098768512:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765133.368344:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098768640:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765133.368364:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765133.368375:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765133/real 1775765133] req@ffff8b23864f2a00 x1862022098768640/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765133.368388:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098768640/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765133.368397:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765133.368400:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765133.368403:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098768640:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765134.263564:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765136.311668:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765138.359622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765138.487576:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765138.487606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765138.487613:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765138.487626:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2599 00000100:00080000:3.0:1775765138.487655:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765138.487665:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765138.487719:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098768768/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765138.487792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765138.487817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765138.487821:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765138.487829:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2599 00000100:00080000:3.0:1775765138.487836:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765138.487844:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765138.487871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765138.487872:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098768768:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765138.487900:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098768896/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765138.487916:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765138.487920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765138.487923:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765138.487928:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2599 00000100:00080000:3.0:1775765138.487972:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765138.487981:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:2.0:1775765138.487989:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:3.0:1775765138.487991:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765138.488019:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098769024/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765138.488038:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765138.488045:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765138.488049:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765138.488055:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765138/real 1775765138] req@ffff8b23864f0700 x1862022098768768/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765138.488063:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2599 00000100:00080000:3.0:1775765138.488078:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765138.488087:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765138.488092:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098768768/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775765138.488101:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1775765138.488109:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765138.488116:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765138.488124:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098768768:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765138.488132:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098769152/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765138.488158:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765138.488167:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098768896:192.168.202.128@tcp:38:kworker.0 00000100:00080000:3.0:1775765138.488181:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765138.488187:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765138.488198:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2599 00000800:00000100:2.0:1775765138.488200:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775765138.488210:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765138.488219:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765138.488242:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765138/real 1775765138] req@ffff8b23864f1180 x1862022098768896/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775765138.488247:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765138.488269:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098769280/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765138.488275:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098768896/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765138.488306:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765138.488313:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765138.488317:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098768896:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765138.488324:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098769024:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765138.488361:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765138.488378:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765138/real 1775765138] req@ffff8b23864f3100 x1862022098769024/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765138.488399:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098769024/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765138.488413:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765138.488436:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765138.488440:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098769024:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765138.488447:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098769152:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765138.488503:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765138.488515:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765138/real 1775765138] req@ffff8b23864f2680 x1862022098769152/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765138.488527:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098769152/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765138.488542:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765138.488545:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765138.488548:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098769152:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765138.488577:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098769280:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765138.488596:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765138.488605:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765138/real 1775765138] req@ffff8b23864f1c00 x1862022098769280/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765138.488616:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098769280/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765138.488624:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765138.488627:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765138.488630:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098769280:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765140.407554:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765142.455573:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765143.607578:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765143.607585:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765143.607589:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765143.607597:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2605 00000100:00080000:1.0:1775765143.607607:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765143.607612:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765143.607644:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098769408/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765143.607681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765143.607684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765143.607685:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765143.607688:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2605 00000100:00080000:1.0:1775765143.607690:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765143.607692:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765143.607699:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765143.607707:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098769536/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765143.607714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765143.607738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765143.607740:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765143.607743:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2605 00000100:00080000:1.0:1775765143.607747:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765143.607750:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765143.607754:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765143.607763:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098769664/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765143.607768:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098769408:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775765143.607772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765143.607775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765143.607777:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765143.607780:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2605 00000100:00080000:1.0:1775765143.607782:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765143.607784:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765143.607787:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765143.607796:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098769792/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765143.607802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765143.607805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765143.607806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765143.607808:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2605 00000100:00080000:1.0:1775765143.607811:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765143.607813:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765143.607817:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765143.607826:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775765143.607830:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098769920/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765143.607863:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765143/real 1775765143] req@ffff8b23864fea00 x1862022098769408/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765143.607878:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098769408/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765143.607886:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765143.607889:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765143.607893:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098769408:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765143.607898:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098769536:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765143.607912:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765143.607920:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765143/real 1775765143] req@ffff8b23864fd180 x1862022098769536/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765143.607928:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098769536/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765143.607933:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765143.607935:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765143.607937:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098769536:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765143.607955:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098769664:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765143.607965:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765143.607971:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765143/real 1775765143] req@ffff8b23864fce00 x1862022098769664/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765143.607978:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098769664/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765143.607983:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765143.607985:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765143.607987:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098769664:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765143.607990:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098769792:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765143.608000:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765143.608005:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765143/real 1775765143] req@ffff8b23864fe680 x1862022098769792/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765143.608012:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098769792/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765143.608017:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765143.608019:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765143.608021:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098769792:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765143.608027:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098769920:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765143.608044:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765143.608054:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765143/real 1775765143] req@ffff8b23864fca80 x1862022098769920/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765143.608067:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098769920/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765143.608076:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765143.608079:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765143.608083:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098769920:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765144.503572:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765146.551555:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765148.599558:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765148.727569:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765148.727578:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765148.727583:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765148.727591:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2610 00000100:00080000:3.0:1775765148.727601:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765148.727607:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765148.727642:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098770048/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765148.727668:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765148.727685:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765148.727688:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765148.727692:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2610 00000100:00080000:3.0:1775765148.727696:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765148.727700:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765148.727710:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765148.727724:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098770176/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765148.727737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765148.727738:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098770048:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765148.727750:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765148.727753:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765148.727757:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2610 00000100:00080000:3.0:1775765148.727760:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765148.727763:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765148.727767:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765148.727777:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098770304/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765148.727786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765148.727789:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765148.727791:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765148.727794:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2610 00000100:00080000:3.0:1775765148.727797:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765148.727800:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765148.727803:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765148.727815:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765148.727816:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098770432/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765148.727827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765148.727830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765148.727832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765148.727837:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2610 00000100:00080000:3.0:1775765148.727839:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765148.727841:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765148.727846:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765148.727854:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098770560/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765148.727871:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765148/real 1775765148] req@ffff8b23864f0a80 x1862022098770048/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765148.727891:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098770048/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765148.727901:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765148.727905:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765148.727909:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098770048:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765148.727930:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098770176:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765148.727948:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765148.727958:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765148/real 1775765148] req@ffff8b23864f0380 x1862022098770176/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765148.727968:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098770176/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765148.727975:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765148.727977:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765148.727980:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098770176:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765148.727984:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098770304:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765148.727997:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765148.728005:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765148/real 1775765148] req@ffff8b23864f1f80 x1862022098770304/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765148.728013:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098770304/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765148.728025:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765148.728028:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765148.728030:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098770304:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765148.728034:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098770432:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765148.728047:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765148.728053:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765148/real 1775765148] req@ffff8b23864f1c00 x1862022098770432/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765148.728062:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098770432/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765148.728068:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765148.728070:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765148.728073:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098770432:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765148.728080:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098770560:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765148.728104:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765148.728119:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765148/real 1775765148] req@ffff8b23864f2680 x1862022098770560/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765148.728134:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098770560/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765148.728145:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765148.728149:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765148.728153:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098770560:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765150.647599:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765152.695632:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765153.847533:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765153.847542:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765153.847546:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765153.847556:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2615 00000100:00080000:3.0:1775765153.847566:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765153.847572:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765153.847609:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098770688/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765153.847654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765153.847657:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765153.847659:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765153.847663:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2615 00000100:00080000:3.0:1775765153.847666:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765153.847669:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765153.847678:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765153.847690:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098770816/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765153.847700:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765153.847704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765153.847706:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765153.847710:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2615 00000100:00080000:3.0:1775765153.847714:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765153.847717:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765153.847722:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765153.847728:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098770688:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765153.847734:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098770944/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765153.847743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765153.847745:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765153.847747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765153.847750:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2615 00000100:00080000:3.0:1775765153.847753:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765153.847756:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765153.847759:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765153.847768:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098771072/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765153.847775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765153.847778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765153.847779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765153.847782:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2615 00000100:00080000:3.0:1775765153.847786:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765153.847789:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765153.847792:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765153.847797:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765153.847802:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098771200/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765153.847837:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765153/real 1775765153] req@ffff8b23864f3100 x1862022098770688/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765153.847855:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098770688/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765153.847864:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765153.847868:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765153.847872:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098770688:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765153.847878:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098770816:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765153.847895:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765153.847904:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765153/real 1775765153] req@ffff8b23864f1180 x1862022098770816/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765153.847913:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098770816/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765153.847919:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765153.847922:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765153.847924:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098770816:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765153.847953:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098770944:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765153.847967:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765153.847974:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765153/real 1775765153] req@ffff8b23864f0700 x1862022098770944/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765153.847983:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098770944/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765153.847989:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765153.847991:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765153.847993:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098770944:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765153.848005:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098771072:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765153.848017:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765153.848023:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765153/real 1775765153] req@ffff8b23864f2a00 x1862022098771072/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765153.848032:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098771072/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765153.848038:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765153.848040:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765153.848042:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098771072:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765153.848046:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098771200:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765153.848066:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765153.848078:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765153/real 1775765153] req@ffff8b23864f0000 x1862022098771200/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765153.848099:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098771200/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765153.848109:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765153.848112:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765153.848116:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098771200:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765154.743598:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765156.791691:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765158.839636:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765158.967688:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765158.967700:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765158.967705:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765158.967715:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2620 00000100:00080000:3.0:1775765158.967729:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765158.967737:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765158.967780:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098771328/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765158.967816:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765158.967820:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765158.967823:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765158.967827:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2620 00000100:00080000:3.0:1775765158.967830:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765158.967834:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765158.967844:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765158.967858:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098771456/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765158.967872:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765158.967876:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765158.967879:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765158.967884:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2620 00000100:00080000:3.0:1775765158.967888:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765158.967892:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765158.967900:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765158.967913:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098771328:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765158.967913:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098771584/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765158.967937:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765158.967941:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765158.967943:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765158.967947:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2620 00000100:00080000:3.0:1775765158.967959:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765158.967963:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765158.967968:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765158.967980:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098771712/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765158.967990:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765158.967993:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765158.967996:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765158.967999:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2620 00000100:00080000:3.0:1775765158.968018:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765158.968025:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:2.0:1775765158.968036:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:3.0:1775765158.968038:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765158.968063:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098771840/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765158.968110:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765158/real 1775765158] req@ffff8b23864f3480 x1862022098771328/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765158.968134:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098771328/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765158.968158:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765158.968163:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765158.968168:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098771328:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765158.968177:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098771456:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765158.968205:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765158.968219:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765158/real 1775765158] req@ffff8b23864f3b80 x1862022098771456/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765158.968240:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098771456/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765158.968248:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765158.968270:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765158.968274:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098771456:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765158.968304:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098771584:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765158.968328:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765158.968340:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765158/real 1775765158] req@ffff8b23864f2d80 x1862022098771584/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765158.968354:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098771584/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765158.968364:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765158.968368:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765158.968372:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098771584:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765158.968379:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098771712:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765158.968398:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765158.968408:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765158/real 1775765158] req@ffff8b23864f0000 x1862022098771712/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765158.968421:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098771712/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765158.968439:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765158.968443:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765158.968447:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098771712:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765158.968453:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098771840:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765158.968530:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765158.968545:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765158/real 1775765158] req@ffff8b23864f2a00 x1862022098771840/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765158.968601:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098771840/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765158.968610:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765158.968623:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765158.968626:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098771840:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765160.887689:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765162.935658:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765164.087695:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765164.087710:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765164.087716:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765164.087731:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2625 00000100:00080000:2.0:1775765164.087749:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765164.087758:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765164.087826:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098771968/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765164.087862:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765164.087865:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765164.087866:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765164.087869:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2625 00000100:00080000:2.0:1775765164.087871:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765164.087875:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765164.087887:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765164.087897:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098772096/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765164.087904:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098771968:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775765164.087910:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765164.087914:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765164.087915:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765164.087919:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2625 00000100:00080000:2.0:1775765164.087922:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765164.087925:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765164.087928:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765164.087938:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098772224/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765164.087945:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765164.087948:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765164.087950:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765164.087953:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2625 00000100:00080000:2.0:1775765164.087972:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765164.087975:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765164.087980:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765164.087988:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098772352/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765164.087995:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765164.088000:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:3.0:1775765164.088000:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775765164.088003:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765164.088009:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2625 00000100:00080000:2.0:1775765164.088011:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765164.088014:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765164.088017:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765164.088025:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098772480/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765164.088101:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765164/real 1775765164] req@ffff8b23b9ea5f80 x1862022098771968/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765164.088138:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022098771968/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765164.088155:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765164.088162:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765164.088168:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098771968:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765164.088210:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098772096:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765164.088255:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765164.088287:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765164/real 1775765164] req@ffff8b23b9ea6680 x1862022098772096/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765164.088296:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022098772096/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765164.088302:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765164.088305:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765164.088307:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098772096:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765164.088311:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098772224:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765164.088325:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765164.088331:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765164/real 1775765164] req@ffff8b23b9ea6a00 x1862022098772224/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765164.088373:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022098772224/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765164.088391:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765164.088395:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765164.088399:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098772224:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765164.088406:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098772352:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765164.088434:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765164.088452:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765164/real 1775765164] req@ffff8b23b9ea4000 x1862022098772352/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765164.088475:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022098772352/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765164.088525:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765164.088542:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765164.088564:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098772352:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765164.088572:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098772480:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765164.088600:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765164.088613:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765164/real 1775765164] req@ffff8b23b9ea5500 x1862022098772480/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765164.088625:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022098772480/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765164.088633:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765164.088637:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765164.088641:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098772480:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765164.983604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765167.031586:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765169.079635:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765169.207553:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765169.207561:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765169.207564:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765169.207572:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2630 00000100:00080000:3.0:1775765169.207581:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765169.207588:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765169.207621:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098772608/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765169.207645:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765169.207653:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765169.207655:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765169.207657:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2630 00000100:00080000:3.0:1775765169.207660:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765169.207663:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765169.207671:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765169.207679:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098772736/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765169.207688:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765169.207693:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098772608:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765169.207693:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765169.207694:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765169.207701:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2630 00000100:00080000:3.0:1775765169.207704:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765169.207707:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765169.207710:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765169.207718:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098772864/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765169.207724:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765169.207726:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765169.207728:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765169.207730:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2630 00000100:00080000:3.0:1775765169.207741:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765169.207744:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765169.207747:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765169.207756:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765169.207757:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098772992/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765169.207765:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765169.207768:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765169.207769:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765169.207773:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2630 00000100:00080000:3.0:1775765169.207775:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765169.207777:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765169.207783:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765169.207793:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098773120/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765169.207804:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765169/real 1775765169] req@ffff8b23864f0700 x1862022098772608/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765169.207820:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098772608/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765169.207834:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765169.207837:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765169.207840:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098772608:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765169.207862:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098772736:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765169.207885:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765169.207898:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765169/real 1775765169] req@ffff8b23864f1180 x1862022098772736/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765169.207912:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098772736/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765169.207921:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765169.207924:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765169.207926:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098772736:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765169.207950:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098772864:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765169.207972:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765169.207983:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765169/real 1775765169] req@ffff8b23864f3100 x1862022098772864/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765169.207994:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098772864/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765169.208002:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765169.208005:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765169.208008:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098772864:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765169.208014:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098772992:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765169.208030:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765169.208038:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765169/real 1775765169] req@ffff8b23864f2680 x1862022098772992/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765169.208054:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098772992/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765169.208066:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765169.208085:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765169.208088:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098772992:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765169.208095:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098773120:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765169.208115:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765169.208136:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765169/real 1775765169] req@ffff8b23864f1c00 x1862022098773120/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765169.208144:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098773120/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765169.208150:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765169.208163:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765169.208165:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098773120:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765171.127735:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765173.175628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765174.327553:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765174.327560:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765174.327564:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765174.327573:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2635 00000100:00080000:3.0:1775765174.327584:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765174.327589:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765174.327623:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098773248/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765174.327647:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765174.327650:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765174.327651:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765174.327654:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2635 00000100:00080000:3.0:1775765174.327656:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765174.327658:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765174.327665:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765174.327673:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098773376/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765174.327681:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765174.327684:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765174.327688:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765174.327691:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2635 00000100:00080000:3.0:1775765174.327693:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765174.327696:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765174.327699:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765174.327707:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098773504/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765174.327715:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765174.327718:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765174.327720:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765174.327726:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2635 00000100:00100000:2.0:1775765174.327727:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098773248:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765174.327729:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765174.327738:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765174.327752:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765174.327762:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098773632/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765174.327769:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765174.327771:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765174.327772:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765174.327775:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2635 00000100:00080000:3.0:1775765174.327777:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765174.327780:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765174.327783:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765174.327789:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098773760/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765174.327838:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765174.327892:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765174/real 1775765174] req@ffff8b23864f1f80 x1862022098773248/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765174.327911:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098773248/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765174.327922:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765174.327926:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765174.327930:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098773248:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765174.327937:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098773376:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765174.327956:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765174.327965:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765174/real 1775765174] req@ffff8b23864f0380 x1862022098773376/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765174.327975:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098773376/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765174.327982:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765174.327985:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765174.327987:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098773376:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765174.328014:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098773504:192.168.202.128@tcp:38:kworker.0 00000800:00000100:2.0:1775765174.328027:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765174.328034:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765174/real 1775765174] req@ffff8b23864f0a80 x1862022098773504/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765174.328049:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098773504/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765174.328055:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765174.328057:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765174.328059:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098773504:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765174.328064:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098773632:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765174.328078:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765174.328085:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765174/real 1775765174] req@ffff8b23864f1c00 x1862022098773632/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765174.328094:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098773632/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765174.328100:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765174.328102:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765174.328104:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098773632:192.168.202.128@tcp:8:kworker.0 00000100:00100000:2.0:1775765174.328109:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098773760:192.168.202.128@tcp:8:kworker.0 00000800:00000100:2.0:1775765174.328120:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765174.328127:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765174/real 1775765174] req@ffff8b23864f2680 x1862022098773760/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765174.328135:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098773760/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765174.328146:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765174.328149:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:2.0:1775765174.328152:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098773760:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765175.223588:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765177.271617:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765179.319565:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765179.447631:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765179.447647:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765179.447655:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765179.447671:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2640 00000100:00080000:2.0:1775765179.447690:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765179.447699:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765179.447759:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098773888/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765179.447807:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765179.447813:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765179.447815:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765179.447821:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2640 00000100:00080000:2.0:1775765179.447827:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765179.447844:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765179.447859:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765179.447863:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098773888:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765179.447883:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098774016/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765179.447912:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765179.447917:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765179.447920:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765179.447928:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2640 00000100:00080000:2.0:1775765179.447936:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765179.447942:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765179.447950:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765179.447950:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765179.447973:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098774144/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765179.447988:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765179.447993:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765179.447996:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765179.448003:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2640 00000100:00080000:2.0:1775765179.448010:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765179.448019:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765179.448020:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765179/real 1775765179] req@ffff8b23b9ea4700 x1862022098773888/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1775765179.448032:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765179.448060:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098774272/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765179.448067:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022098773888/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765179.448079:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765179.448099:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00080000:2.0:1775765179.448103:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775765179.448107:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098773888:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775765179.448112:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765179.448122:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765179.448132:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2640 00000100:00080000:2.0:1775765179.448140:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765179.448152:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765179.448152:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098774016:192.168.202.128@tcp:38:kworker.0 00000008:00100000:2.0:1775765179.448178:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765179.448193:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765179.448203:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098774400/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765179.448216:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765179/real 1775765179] req@ffff8b23b9ea4a80 x1862022098774016/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765179.448245:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022098774016/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765179.448255:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765179.448260:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765179.448265:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098774016:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765179.448275:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098774144:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765179.448300:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765179.448322:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765179/real 1775765179] req@ffff8b23b9ea7100 x1862022098774144/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765179.448335:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022098774144/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765179.448349:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765179.448353:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765179.448356:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098774144:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765179.448364:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098774272:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765179.448385:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765179.448414:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765179/real 1775765179] req@ffff8b23b9ea4380 x1862022098774272/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765179.448427:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022098774272/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765179.448446:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765179.448449:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765179.448476:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098774272:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765179.448495:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098774400:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765179.448516:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765179.448528:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765179/real 1775765179] req@ffff8b23b9ea4e00 x1862022098774400/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765179.448549:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022098774400/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765179.448557:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765179.448561:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765179.448564:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098774400:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765181.367613:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765183.415588:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765184.567595:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765184.567605:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765184.567610:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765184.567620:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2645 00000100:00080000:3.0:1775765184.567632:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765184.567640:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765184.567682:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098774528/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765184.567719:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765184.567723:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765184.567726:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765184.567731:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2645 00000100:00080000:3.0:1775765184.567735:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765184.567741:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765184.567751:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765184.567768:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098774656/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765184.567787:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765184.567800:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765184.567803:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765184.567813:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2645 00000100:00080000:3.0:1775765184.567820:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765184.567845:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765184.567853:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765184.567874:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098774784/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765184.567889:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765184.567894:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765184.567896:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765184.567901:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2645 00000100:00080000:3.0:1775765184.567907:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765184.567912:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765184.567918:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765184.567949:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098774912/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765184.567961:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765184.567972:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765184.567975:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765184.567984:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2645 00000100:00080000:3.0:1775765184.567990:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765184.568019:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765184.568031:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765184.568050:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098775040/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765185.463626:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765187.511616:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775765187.639809:0:2364:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775765187.639865:0:2364:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775765187.639887:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765187.639933:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765187.639942:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765187.639950:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765187.639956:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00010000:00010000:2.0:1775765189.559535:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765189.687825:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098775168:192.168.202.128@tcp:250:kworker.0 00000800:00000100:3.0:1775765189.687953:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765189.688029:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765189/real 1775765189] req@ffff8b2382dad880 x1862022098775168/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765189.688056:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098775168/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765189.688071:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765189.688077:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765189.688082:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098775168:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765189.688092:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098775296:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765189.688123:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765189.688140:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765189/real 1775765189] req@ffff8b2382daca80 x1862022098775296/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765189.688154:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098775296/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765189.688165:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765189.688169:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765189.688172:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098775296:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765189.688179:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098775424:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765189.688206:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765189.688218:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765189/real 1775765189] req@ffff8b2382daf100 x1862022098775424/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765189.688239:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098775424/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765189.688727:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765189.688732:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765189.688737:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098775424:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765189.688791:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098775552:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765189.688829:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765189.688846:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765189/real 1775765189] req@ffff8b2382dac000 x1862022098775552/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765189.688861:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098775552/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765189.688870:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765189.688875:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765189.688880:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098775552:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765189.688888:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098775680:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765189.688923:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765189.688936:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765189/real 1775765189] req@ffff8b2382dac380 x1862022098775680/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765189.688950:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098775680/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765189.688965:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765189.688969:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765189.688972:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098775680:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765191.607609:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765193.655557:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765194.807601:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765194.807615:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765194.807622:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765194.807636:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2656 00000100:00080000:1.0:1775765194.807653:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765194.807663:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765194.807718:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098775808/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765194.807774:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765194.807780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765194.807784:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765194.807791:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2656 00000100:00080000:1.0:1775765194.807797:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765194.807804:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765194.807814:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098775808:192.168.202.128@tcp:250:kworker.0 00000008:00100000:1.0:1775765194.807819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765194.807844:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098775936/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765194.807864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765194.807869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765194.807872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765194.807880:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2656 00000100:00080000:1.0:1775765194.807888:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775765194.807889:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775765194.807907:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765194.807915:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765194.807933:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098776064/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765194.807946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765194.807951:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765194.807955:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775765194.807962:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765194/real 1775765194] req@ffff8b23864fdf80 x1862022098775808/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765194.807970:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2656 00000100:00080000:1.0:1775765194.807987:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775765194.807998:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098775808/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765194.808008:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:3.0:1775765194.808008:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765194.808027:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000008:00100000:1.0:1775765194.808030:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765194.808032:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098775808:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765194.808070:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098775936:192.168.202.128@tcp:38:kworker.0 00000100:00100000:1.0:1775765194.808071:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098776192/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765194.808100:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765194.808105:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:3.0:1775765194.808109:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775765194.808111:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765194.808120:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2656 00000100:00100000:3.0:1775765194.808124:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765194/real 1775765194] req@ffff8b23864fc380 x1862022098775936/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765194.808133:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765194.808163:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765194.808165:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098775936/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775765194.808178:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775765194.808178:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765194.808188:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765194.808193:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098775936:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765194.808200:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098776064:192.168.202.128@tcp:38:kworker.0 00000100:00100000:1.0:1775765194.808213:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098776320/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765194.808229:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765194.808269:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765194/real 1775765194] req@ffff8b23864fed80 x1862022098776064/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765194.808284:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098776064/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765194.808296:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765194.808301:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765194.808304:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098776064:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765194.808322:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098776192:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765194.808345:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765194.808357:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765194/real 1775765194] req@ffff8b23864fc000 x1862022098776192/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765194.808370:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098776192/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765194.808379:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765194.808382:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765194.808385:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098776192:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765194.808392:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098776320:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765194.808413:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765194.808425:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765194/real 1775765194] req@ffff8b23864ff800 x1862022098776320/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765194.808434:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098776320/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765194.808439:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765194.808441:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765194.808443:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098776320:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765195.703584:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765197.751579:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765199.799668:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765199.927561:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765199.927570:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765199.927574:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765199.927584:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2661 00000100:00080000:1.0:1775765199.927602:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765199.927609:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765199.927646:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098776448/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765199.927674:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765199.927678:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765199.927680:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765199.927683:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2661 00000100:00080000:1.0:1775765199.927687:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765199.927691:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765199.927699:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765199.927713:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098776576/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765199.927722:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098776448:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775765199.927726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765199.927731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765199.927733:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765199.927738:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2661 00000100:00080000:1.0:1775765199.927742:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765199.927746:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765199.927752:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765199.927763:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098776704/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765199.927772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765199.927776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1775765199.927779:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775765199.927780:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765199.927787:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2661 00000100:00080000:1.0:1775765199.927791:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765199.927795:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765199.927800:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765199.927812:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098776832/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765199.927818:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765199/real 1775765199] req@ffff8b23864ff800 x1862022098776448/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765199.927827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765199.927833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765199.927836:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775765199.927840:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098776448/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765199.927841:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2661 00000100:00080000:1.0:1775765199.927848:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765199.927853:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765199.927854:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:3.0:1775765199.927860:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000008:00100000:1.0:1775765199.927863:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765199.927864:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098776448:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775765199.927882:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098776960/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765199.927901:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098776576:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765199.927928:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765199.927942:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765199/real 1775765199] req@ffff8b23864fc000 x1862022098776576/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765199.927951:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098776576/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765199.927961:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765199.927964:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765199.927966:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098776576:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765199.927970:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098776704:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765199.927992:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765199.928000:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765199/real 1775765199] req@ffff8b23864fed80 x1862022098776704/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765199.928007:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098776704/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765199.928012:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765199.928014:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765199.928016:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098776704:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765199.928020:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098776832:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765199.928032:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765199.928037:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765199/real 1775765199] req@ffff8b23864fc380 x1862022098776832/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765199.928045:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098776832/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765199.928049:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765199.928051:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765199.928053:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098776832:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765199.928057:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098776960:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765199.928073:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765199.928078:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765199/real 1775765199] req@ffff8b23864fdf80 x1862022098776960/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765199.928088:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098776960/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765199.928096:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765199.928099:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765199.928103:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098776960:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765201.847685:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765203.895563:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765205.047528:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765205.047536:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765205.047540:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765205.047549:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2666 00000100:00080000:3.0:1775765205.047558:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765205.047564:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765205.047606:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098777088/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765205.047628:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765205.047631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765205.047633:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765205.047635:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2666 00000100:00080000:3.0:1775765205.047638:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765205.047641:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765205.047648:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765205.047656:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098777216/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765205.047665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765205.047667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765205.047669:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765205.047672:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2666 00000100:00080000:3.0:1775765205.047675:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765205.047679:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765205.047682:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765205.047689:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098777344/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765205.047695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765205.047697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765205.047699:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765205.047702:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2666 00000100:00080000:3.0:1775765205.047704:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765205.047706:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765205.047710:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765205.047721:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098777472/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765205.047743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765205.047746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765205.047747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765205.047750:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2666 00000100:00080000:3.0:1775765205.047753:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765205.047756:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765205.047758:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765205.047771:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098777600/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765205.943635:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765207.991588:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765210.039602:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765210.167608:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765210.167622:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765210.167628:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765210.167640:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2671 00000100:00080000:3.0:1775765210.167654:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765210.167663:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765210.167751:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098777728/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765210.167802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765210.167809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765210.167812:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765210.167818:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2671 00000100:00080000:3.0:1775765210.167824:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765210.167830:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765210.167843:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765210.167867:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098777856/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765210.167884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765210.167902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765210.167904:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765210.167912:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2671 00000100:00080000:3.0:1775765210.167922:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765210.167933:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765210.167950:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765210.167969:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098777984/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765210.168005:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765210.168012:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765210.168015:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765210.168023:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2671 00000100:00080000:3.0:1775765210.168033:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765210.168050:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765210.168068:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765210.168098:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098778112/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765210.168144:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765210.168168:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765210.168178:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765210.168189:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2671 00000100:00080000:3.0:1775765210.168201:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765210.168222:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765210.168245:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765210.168287:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098778240/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765212.087567:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765214.135643:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765215.287589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765215.287600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765215.287605:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765215.287622:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2676 00000100:00080000:3.0:1775765215.287636:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765215.287643:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765215.287695:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098778368/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765215.287722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765215.287726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765215.287727:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765215.287730:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2676 00000100:00080000:3.0:1775765215.287732:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765215.287735:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765215.287744:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765215.287754:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098778496/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765215.287763:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765215.287769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765215.287771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765215.287774:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2676 00000100:00080000:3.0:1775765215.287777:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765215.287781:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765215.287785:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765215.287795:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098778624/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765215.287801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765215.287803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765215.287804:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765215.287807:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2676 00000100:00080000:3.0:1775765215.287809:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765215.287811:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765215.287814:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765215.287823:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098778752/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765215.287829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765215.287831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765215.287832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765215.287835:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2676 00000100:00080000:3.0:1775765215.287837:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765215.287840:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765215.287843:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765215.287850:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098778880/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765216.183586:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765218.231649:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765220.279606:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765220.407592:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765220.407603:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765220.407609:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765220.407620:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2681 00000100:00080000:1.0:1775765220.407633:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765220.407640:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765220.407687:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098779008/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765220.407717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765220.407721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765220.407722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765220.407725:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2681 00000100:00080000:1.0:1775765220.407727:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765220.407730:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765220.407739:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765220.407749:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098779136/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765220.407758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765220.407761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765220.407763:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765220.407766:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2681 00000100:00080000:1.0:1775765220.407770:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765220.407773:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765220.407779:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765220.407790:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098779264/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765220.407819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765220.407822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765220.407824:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765220.407828:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2681 00000100:00080000:1.0:1775765220.407830:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765220.407833:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765220.407836:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765220.407846:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098779392/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765220.407852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765220.407855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765220.407857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765220.407861:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2681 00000100:00080000:1.0:1775765220.407864:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765220.407866:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765220.407869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765220.407887:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098779520/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765222.327646:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765224.375551:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765225.527577:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765225.527591:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765225.527597:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765225.527597:0:68331:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b2384b86000 already not connected (conn 56, was 1): DISCONN 00000100:00080000:1.0:1775765225.527616:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2686 00010000:00020000:3.0:1775765225.527620:0:68331:0:(ldlm_request.c:123:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1775764895, 330s ago), entering recovery for lustre-MDT0000_UUID@192.168.202.128@tcp ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5ec00/0x73cfd86cfc17c429 lrc: 4/1,0 mode: --/CR res: [0x200003ab4:0x24:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x57b8ff0e030534a6 expref: -99 pid: 68331 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000100:00080000:1.0:1775765225.527631:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765225.527649:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765225.527704:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098779648/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765225.527747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765225.527752:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765225.527755:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765225.527760:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2686 00000100:00080000:1.0:1775765225.527764:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765225.527768:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765225.527779:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765225.527794:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098779776/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765225.527812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765225.527817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765225.527819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765225.527826:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2686 00000100:00080000:1.0:1775765225.527829:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765225.527833:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765225.527838:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765225.527852:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098779904/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765225.527875:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765225.527880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765225.527882:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765225.527888:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2686 00000100:00080000:1.0:1775765225.527891:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765225.527896:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765225.527901:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765225.527916:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098780032/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765225.527927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765225.527931:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765225.527933:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765225.527938:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2686 00000100:00080000:1.0:1775765225.527957:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765225.527992:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765225.528006:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765225.528029:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098780160/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765226.423614:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765228.471623:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765230.519597:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765230.647775:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098780288:192.168.202.128@tcp:250:kworker.0 00000800:00000100:3.0:1775765230.647850:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765230.647904:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765230/real 1775765230] req@ffff8b2382dae680 x1862022098780288/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765230.647971:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098780288/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765230.647986:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765230.647991:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765230.647998:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098780288:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765230.648025:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098780416:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765230.648047:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765230.648056:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765230/real 1775765230] req@ffff8b2382dad500 x1862022098780416/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765230.648065:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098780416/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765230.648075:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765230.648077:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765230.648080:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098780416:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765230.648083:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098780544:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765230.648095:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765230.648101:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765230/real 1775765230] req@ffff8b2382daea00 x1862022098780544/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765230.648108:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098780544/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765230.648113:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765230.648115:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765230.648117:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098780544:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765230.648121:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098780672:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765230.648131:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765230.648137:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765230/real 1775765230] req@ffff8b2382daed80 x1862022098780672/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765230.648145:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098780672/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765230.648149:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765230.648151:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765230.648154:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098780672:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765230.648158:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098780800:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765230.648168:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765230.648173:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765230/real 1775765230] req@ffff8b2382dace00 x1862022098780800/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765230.648180:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098780800/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765230.648185:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765230.648187:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765230.648189:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098780800:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765232.567531:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765234.615617:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765235.767898:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098780928:192.168.202.128@tcp:250:kworker.0 00000800:00000100:3.0:1775765235.767986:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765235.768048:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765235/real 1775765235] req@ffff8b2382dafb80 x1862022098780928/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765235.768074:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098780928/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765235.768101:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765235.768112:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765235.768117:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098780928:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765235.768147:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098781056:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765235.768180:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765235.768212:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765235/real 1775765235] req@ffff8b2382dadf80 x1862022098781056/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765235.768228:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098781056/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765235.768238:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765235.768242:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765235.768246:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098781056:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765235.768255:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098781184:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765235.768280:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765235.768297:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765235/real 1775765235] req@ffff8b2382daf480 x1862022098781184/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765235.768316:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098781184/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765235.768329:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765235.768334:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765235.768356:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098781184:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765235.768366:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098781312:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765235.768399:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765235.768417:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765235/real 1775765235] req@ffff8b2382dac700 x1862022098781312/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765235.768432:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098781312/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765235.768441:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765235.768444:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765235.768448:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098781312:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765235.768534:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098781440:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765235.768561:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765235.768590:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765235/real 1775765235] req@ffff8b2382dafb80 x1862022098781440/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765235.768602:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098781440/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765235.768611:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765235.768614:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765235.768618:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098781440:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765236.663628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765238.711560:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765240.759675:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765240.887661:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765240.887670:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765240.887673:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765240.887682:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2702 00000100:00080000:3.0:1775765240.887692:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765240.887697:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765240.887731:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098781568/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765240.887766:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765240.887770:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765240.887771:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765240.887774:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2702 00000100:00080000:3.0:1775765240.887776:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765240.887779:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765240.887786:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765240.887796:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098781696/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765240.887805:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765240.887807:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765240.887809:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765240.887812:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2702 00000100:00080000:3.0:1775765240.887814:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765240.887831:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765240.887834:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765240.887841:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098781824/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765240.887850:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765240.887852:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765240.887853:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765240.887857:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2702 00000100:00080000:3.0:1775765240.887860:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765240.887869:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765240.887876:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765240.887901:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098781952/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765240.887907:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765240.887909:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765240.887910:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765240.887914:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2702 00000100:00080000:3.0:1775765240.887916:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765240.887918:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765240.887921:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765240.887935:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098782080/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765242.807574:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765244.855611:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765246.007521:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765246.007530:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765246.007535:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765246.007545:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2707 00000100:00080000:3.0:1775765246.007556:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765246.007563:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765246.007598:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098782208/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765246.007627:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765246.007631:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765246.007633:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765246.007637:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2707 00000100:00080000:3.0:1775765246.007641:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765246.007645:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765246.007655:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765246.007670:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098782336/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765246.007683:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765246.007689:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765246.007692:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765246.007702:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2707 00000100:00080000:3.0:1775765246.007706:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765246.007710:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765246.007715:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765246.007737:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098782464/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765246.007748:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765246.007752:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765246.007754:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765246.007759:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2707 00000100:00080000:3.0:1775765246.007763:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765246.007785:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765246.007792:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765246.007805:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098782592/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765246.007822:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765246.007825:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765246.007827:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765246.007831:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2707 00000100:00080000:3.0:1775765246.007835:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765246.007840:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765246.007846:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765246.007862:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098782720/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765246.903720:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765248.951634:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765250.999589:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765251.127568:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765251.127577:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765251.127580:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765251.127589:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2712 00000100:00080000:3.0:1775765251.127599:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765251.127604:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765251.127639:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098782848/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765251.127668:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765251.127671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765251.127673:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765251.127676:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2712 00000100:00080000:3.0:1775765251.127678:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765251.127680:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765251.127688:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765251.127698:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098782976/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765251.127705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765251.127708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765251.127709:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765251.127711:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2712 00000100:00080000:3.0:1775765251.127713:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765251.127716:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765251.127718:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765251.127742:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098783104/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765251.127749:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765251.127751:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765251.127753:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765251.127756:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2712 00000100:00080000:3.0:1775765251.127759:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765251.127762:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765251.127766:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765251.127773:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098783232/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765251.127783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765251.127789:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765251.127790:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765251.127794:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2712 00000100:00080000:3.0:1775765251.127796:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765251.127798:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765251.127801:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765251.127809:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098783360/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765253.047571:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765254.200013:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765251/real 1775765254] req@ffff8b23864f2680 x1862022098783232/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765254.200045:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098783232/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765254.200060:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765254.200066:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765254.200071:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098783232:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765254.200081:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765251/real 1775765254] req@ffff8b23864f1c00 x1862022098783360/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765254.200096:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098783360/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765254.200105:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765254.200109:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765254.200112:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098783360:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765254.200120:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765251/real 1775765254] req@ffff8b23864f2a00 x1862022098782848/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765254.200137:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098782848/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765254.200162:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765254.200167:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765254.200171:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098782848:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765254.200179:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765251/real 1775765254] req@ffff8b23864f0700 x1862022098782976/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765254.200194:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098782976/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765254.200203:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765254.200207:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765254.200211:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098782976:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765254.200219:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765251/real 1775765254] req@ffff8b23864f0000 x1862022098783104/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765254.200234:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098783104/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765254.200242:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765254.200245:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765254.200249:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098783104:192.168.202.128@tcp:38:kworker.0 00010000:00010000:2.0:1775765255.095616:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765256.247718:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098783488:192.168.202.128@tcp:250:kworker.0 00000800:00000100:3.0:1775765256.247781:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765256.247818:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765256/real 1775765256] req@ffff8b2382dace00 x1862022098783488/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765256.247837:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098783488/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765256.247857:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765256.247865:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765256.247872:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098783488:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765256.247897:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098783616:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765256.247925:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765256.247942:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765256/real 1775765256] req@ffff8b2382daed80 x1862022098783616/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765256.247957:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098783616/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765256.247965:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765256.247969:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765256.247973:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098783616:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765256.247979:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098783744:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765256.248009:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765256.248028:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765256/real 1775765256] req@ffff8b2382daea00 x1862022098783744/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765256.248048:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098783744/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765256.248056:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765256.248060:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765256.248063:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098783744:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765256.248070:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098783872:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765256.248098:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765256.248110:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765256/real 1775765256] req@ffff8b2382dad500 x1862022098783872/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765256.248121:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098783872/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765256.248127:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765256.248130:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765256.248132:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098783872:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765256.248150:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098784000:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765256.248171:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765256.248184:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765256/real 1775765256] req@ffff8b2382dae680 x1862022098784000/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765256.248196:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098784000/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765256.248211:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765256.248214:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765256.248219:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098784000:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765257.143575:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765259.191600:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765261.239734:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765261.367688:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765261.367699:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765261.367704:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765261.367715:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2722 00000100:00080000:3.0:1775765261.367729:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765261.367736:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765261.367781:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098784128/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765261.367822:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765261.367828:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765261.367832:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765261.367838:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2722 00000100:00080000:3.0:1775765261.367847:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765261.367855:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765261.367870:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765261.367916:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098784256/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765261.367938:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765261.367943:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765261.367957:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765261.367962:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2722 00000100:00080000:3.0:1775765261.367968:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765261.367979:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765261.367986:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765261.368002:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098784384/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765261.368011:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765261.368015:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765261.368016:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765261.368022:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2722 00000100:00080000:3.0:1775765261.368026:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765261.368030:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765261.368035:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765261.368051:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098784512/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765261.368070:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765261.368083:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765261.368088:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765261.368098:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2722 00000100:00080000:3.0:1775765261.368101:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765261.368112:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765261.368123:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765261.368138:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098784640/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765263.287680:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765265.335619:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765266.487513:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765266.487521:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765266.487525:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765266.487533:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2727 00000100:00080000:1.0:1775765266.487543:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765266.487549:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765266.487594:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098784768/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765266.487618:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765266.487621:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765266.487622:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765266.487624:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2727 00000100:00080000:1.0:1775765266.487627:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765266.487629:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765266.487636:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765266.487645:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098784896/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765266.487654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765266.487657:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765266.487659:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765266.487662:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2727 00000100:00080000:1.0:1775765266.487664:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765266.487667:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765266.487670:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765266.487680:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098785024/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765266.487687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765266.487689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765266.487690:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765266.487693:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2727 00000100:00080000:1.0:1775765266.487695:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765266.487697:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765266.487700:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765266.487709:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098785152/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765266.487715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765266.487717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765266.487718:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765266.487720:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2727 00000100:00080000:1.0:1775765266.487723:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765266.487725:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765266.487728:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765266.487739:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098785280/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765267.383718:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765269.431575:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765271.479783:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765271.607579:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765271.607593:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765271.607598:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765271.607609:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2733 00000100:00080000:1.0:1775765271.607623:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765271.607631:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765271.607676:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098785408/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765271.607717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765271.607725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765271.607727:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765271.607731:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2733 00000100:00080000:1.0:1775765271.607735:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765271.607739:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765271.607749:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765271.607764:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098785536/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765271.607778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765271.607783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765271.607785:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765271.607790:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2733 00000100:00080000:1.0:1775765271.607795:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765271.607799:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765271.607804:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765271.607823:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098785664/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765271.607834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765271.607838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765271.607841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765271.607845:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2733 00000100:00080000:1.0:1775765271.607849:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765271.607853:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765271.607860:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765271.607871:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098785792/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765271.607881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765271.607885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765271.607886:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765271.607889:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2733 00000100:00080000:1.0:1775765271.607894:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765271.607897:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765271.607902:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765271.607922:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098785920/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765273.527575:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765275.575639:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765276.727549:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765276.727557:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765276.727561:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765276.727587:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2738 00000100:00080000:1.0:1775765276.727600:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765276.727608:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765276.727650:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098786048/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765276.727682:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765276.727686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765276.727692:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765276.727695:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2738 00000100:00080000:1.0:1775765276.727699:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765276.727703:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765276.727711:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765276.727741:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098786176/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765276.727751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765276.727754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765276.727758:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765276.727765:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2738 00000100:00080000:1.0:1775765276.727767:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765276.727770:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765276.727773:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765276.727824:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098786304/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765276.727832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765276.727834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765276.727836:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765276.727839:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2738 00000100:00080000:1.0:1775765276.727843:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765276.727845:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765276.727849:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765276.727861:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098786432/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765276.727874:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765276.727877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765276.727881:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765276.727886:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2738 00000100:00080000:1.0:1775765276.727891:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765276.727896:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765276.727901:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765276.727910:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098786560/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765277.623579:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765279.671600:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765281.719628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765281.847644:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765281.847667:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765281.847672:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765281.847685:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2743 00000100:00080000:3.0:1775765281.847717:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765281.847724:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765281.847775:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098786688/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765281.847804:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765281.847807:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765281.847809:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765281.847812:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2743 00000100:00080000:3.0:1775765281.847815:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765281.847818:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765281.847827:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765281.847839:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098786816/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765281.847851:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765281.847856:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765281.847859:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765281.847868:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2743 00000100:00080000:3.0:1775765281.847871:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765281.847875:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765281.847879:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765281.847892:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098786944/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765281.847900:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765281.847903:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765281.847905:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765281.847908:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2743 00000100:00080000:3.0:1775765281.847910:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765281.847913:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765281.847917:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765281.847936:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098787072/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765281.847945:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765281.847949:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765281.847952:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765281.847956:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2743 00000100:00080000:3.0:1775765281.847960:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765281.847963:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765281.847967:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765281.847975:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098787200/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765283.767562:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765285.815545:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765286.967524:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765286.967533:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765286.967537:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765286.967546:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2748 00000100:00080000:3.0:1775765286.967557:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765286.967563:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765286.967602:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098787328/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765286.967629:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765286.967632:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765286.967634:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765286.967637:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2748 00000100:00080000:3.0:1775765286.967640:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765286.967643:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765286.967651:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765286.967663:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098787456/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765286.967674:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765286.967678:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765286.967679:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765286.967683:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2748 00000100:00080000:3.0:1775765286.967685:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765286.967688:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765286.967694:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765286.967711:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098787584/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765286.967726:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765286.967730:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765286.967733:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765286.967739:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2748 00000100:00080000:3.0:1775765286.967744:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765286.967749:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765286.967756:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765286.967775:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098787712/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765286.967786:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765286.967789:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765286.967791:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765286.967795:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2748 00000100:00080000:3.0:1775765286.967799:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765286.967805:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765286.967815:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765286.967833:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098787840/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765287.863604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765289.911713:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765291.959581:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765292.087530:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765292.087538:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765292.087542:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765292.087550:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2753 00000100:00080000:1.0:1775765292.087560:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765292.087565:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765292.087609:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098787968/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765292.087632:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765292.087635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765292.087636:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765292.087639:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2753 00000100:00080000:1.0:1775765292.087641:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765292.087644:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765292.087652:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765292.087663:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098788096/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765292.087672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765292.087674:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765292.087676:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765292.087681:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2753 00000100:00080000:1.0:1775765292.087685:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765292.087691:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765292.087695:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765292.087734:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098788224/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765292.087748:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765292.087753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765292.087756:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765292.087763:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2753 00000100:00080000:1.0:1775765292.087795:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765292.087799:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765292.087808:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765292.087825:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098788352/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765292.087837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765292.087840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765292.087859:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765292.087863:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2753 00000100:00080000:1.0:1775765292.087867:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765292.087872:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765292.087879:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765292.087894:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098788480/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765294.007637:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765296.055587:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765297.207559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765297.207569:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765297.207574:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765297.207602:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2758 00000100:00080000:3.0:1775765297.207617:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765297.207626:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765297.207686:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098788608/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765297.207717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765297.207721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765297.207723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765297.207727:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2758 00000100:00080000:3.0:1775765297.207730:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765297.207734:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765297.207743:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765297.207756:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098788736/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765297.207767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765297.207771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765297.207773:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765297.207787:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2758 00000100:00080000:3.0:1775765297.207791:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765297.207802:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765297.207811:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765297.207822:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098788864/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765297.207832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765297.207837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765297.207839:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765297.207842:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2758 00000100:00080000:3.0:1775765297.207846:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765297.207849:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765297.207853:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765297.207864:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098788992/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765297.207874:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765297.207881:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765297.207887:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765297.207892:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2758 00000100:00080000:3.0:1775765297.207897:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765297.207900:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765297.207906:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765297.207922:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098789120/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765298.103558:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765300.151626:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765302.199544:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765302.327668:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098789248:192.168.202.128@tcp:250:kworker.0 00000800:00000100:3.0:1775765302.327731:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765302.327801:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765302/real 1775765302] req@ffff8b2382dac380 x1862022098789248/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765302.327817:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098789248/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765302.327830:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765302.327833:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765302.327836:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098789248:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765302.327857:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098789376:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765302.327872:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765302.327879:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765302/real 1775765302] req@ffff8b2382dac000 x1862022098789376/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765302.327887:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098789376/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765302.327892:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765302.327895:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765302.327897:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098789376:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765302.327900:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098789504:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765302.327910:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765302.327915:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765302/real 1775765302] req@ffff8b2382daf100 x1862022098789504/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765302.327922:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098789504/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765302.327928:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765302.327930:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765302.327933:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098789504:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765302.327940:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098789632:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765302.327959:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765302.327968:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765302/real 1775765302] req@ffff8b2382daca80 x1862022098789632/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765302.327981:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098789632/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765302.327990:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765302.327993:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765302.327996:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098789632:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765302.328000:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098789760:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765302.328015:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765302.328022:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765302/real 1775765302] req@ffff8b2382dad880 x1862022098789760/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765302.328029:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098789760/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765302.328034:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765302.328036:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765302.328038:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098789760:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765304.247576:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765306.295568:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765307.447758:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098789888:192.168.202.128@tcp:250:kworker.0 00000800:00000100:3.0:1775765307.447828:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765307.447878:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765307/real 1775765307] req@ffff8b2382dad880 x1862022098789888/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765307.447895:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098789888/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765307.447905:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765307.447910:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765307.447924:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098789888:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765307.447932:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098790016:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765307.447952:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765307.447962:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765307/real 1775765307] req@ffff8b2382daca80 x1862022098790016/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765307.447972:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098790016/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765307.447978:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765307.447981:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765307.447983:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098790016:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765307.447988:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098790144:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765307.448000:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765307.448006:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765307/real 1775765307] req@ffff8b2382daf100 x1862022098790144/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765307.448015:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098790144/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765307.448027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765307.448030:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765307.448032:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098790144:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765307.448055:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098790272:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765307.448069:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765307.448080:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765307/real 1775765307] req@ffff8b2382dac000 x1862022098790272/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765307.448095:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098790272/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765307.448105:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765307.448109:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765307.448113:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098790272:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765307.448121:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098790400:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765307.448161:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765307.448175:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765307/real 1775765307] req@ffff8b2382dac380 x1862022098790400/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765307.448191:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098790400/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765307.448198:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765307.448242:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765307.448246:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098790400:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765308.343618:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765310.391592:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765312.439620:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765312.567571:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765312.567584:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765312.567589:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765312.567600:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2773 00000100:00080000:3.0:1775765312.567614:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765312.567620:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765312.567684:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098790528/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765312.567720:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765312.567724:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765312.567737:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765312.567740:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2773 00000100:00080000:3.0:1775765312.567744:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765312.567747:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765312.567757:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765312.567768:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098790656/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765312.567778:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765312.567782:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765312.567784:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765312.567791:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2773 00000100:00080000:3.0:1775765312.567794:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765312.567798:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765312.567801:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765312.567809:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098790784/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765312.567815:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765312.567819:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765312.567820:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765312.567823:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2773 00000100:00080000:3.0:1775765312.567824:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765312.567827:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765312.567830:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765312.567837:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098790912/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765312.567842:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765312.567845:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765312.567846:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765312.567849:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2773 00000100:00080000:3.0:1775765312.567851:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765312.567853:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765312.567856:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765312.567865:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098791040/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765314.487640:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765316.535634:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765317.687588:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765317.687598:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765317.687603:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765317.687614:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2779 00000100:00080000:3.0:1775765317.687626:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765317.687634:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765317.687678:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098791168/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765317.687709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765317.687714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765317.687716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765317.687720:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2779 00000100:00080000:3.0:1775765317.687723:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765317.687727:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765317.687738:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765317.687754:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098791296/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765317.687779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765317.687782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765317.687784:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765317.687789:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2779 00000100:00080000:3.0:1775765317.687793:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765317.687796:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765317.687801:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765317.687813:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098791424/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765317.687823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765317.687827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765317.687829:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765317.687833:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2779 00000100:00080000:3.0:1775765317.687836:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765317.687840:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765317.687844:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765317.687854:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098791552/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765317.687863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765317.687866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765317.687868:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765317.687873:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2779 00000100:00080000:3.0:1775765317.687877:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765317.687880:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765317.687884:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765317.687898:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098791680/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765318.583735:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765320.631579:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775765320.759719:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775765320.759792:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775765320.759813:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775765320.759881:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775765320.759893:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775765320.759902:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775765320.759910:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00010000:00010000:2.0:1775765322.679578:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765322.807599:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765322.807607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765322.807610:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765322.807617:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2784 00000100:00080000:1.0:1775765322.807626:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765322.807631:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765322.807661:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098791808/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765322.807689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765322.807693:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765322.807696:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765322.807700:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2784 00000100:00080000:1.0:1775765322.807703:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765322.807707:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765322.807718:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765322.807734:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098791936/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765322.807746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765322.807751:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765322.807754:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765322.807759:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2784 00000100:00080000:1.0:1775765322.807763:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765322.807767:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765322.807773:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765322.807785:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098792064/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765322.807798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765322.807803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765322.807806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765322.807815:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2784 00000100:00080000:1.0:1775765322.807820:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765322.807835:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765322.807848:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765322.807882:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098792192/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765322.807893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765322.807896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765322.807899:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765322.807904:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2784 00000100:00080000:1.0:1775765322.807908:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765322.807913:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765322.807920:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765322.807934:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098792320/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765324.727628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765326.775576:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765327.927543:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765327.927551:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765327.927555:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765327.927563:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2789 00000100:00080000:3.0:1775765327.927572:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765327.927577:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765327.927611:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098792448/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765327.927634:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765327.927637:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765327.927639:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765327.927643:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2789 00000100:00080000:3.0:1775765327.927646:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765327.927651:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765327.927660:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765327.927674:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098792576/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765327.927697:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765327.927701:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765327.927702:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765327.927706:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2789 00000100:00080000:3.0:1775765327.927709:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765327.927712:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765327.927717:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765327.927726:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098792704/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765327.927734:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765327.927737:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765327.927739:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765327.927742:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2789 00000100:00080000:3.0:1775765327.927745:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765327.927748:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765327.927752:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765327.927760:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098792832/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765327.927768:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765327.927771:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765327.927773:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765327.927776:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2789 00000100:00080000:3.0:1775765327.927779:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765327.927782:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765327.927786:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765327.927794:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098792960/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765328.823644:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765330.871641:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765332.919635:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765333.047626:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765333.047639:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765333.047647:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765333.047662:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2794 00000100:00080000:3.0:1775765333.047679:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765333.047688:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765333.047748:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098793088/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765333.047787:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765333.047792:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765333.047794:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765333.047798:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2794 00000100:00080000:3.0:1775765333.047801:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765333.047805:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765333.047818:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765333.047834:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098793216/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765333.047849:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765333.047855:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765333.047858:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765333.047865:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2794 00000100:00080000:3.0:1775765333.047868:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765333.047872:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765333.047879:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765333.047893:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098793344/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765333.047903:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765333.047907:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765333.047908:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765333.047913:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2794 00000100:00080000:3.0:1775765333.047917:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765333.047920:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765333.047925:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765333.047936:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098793472/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765333.047946:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765333.047949:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765333.047951:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765333.047955:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2794 00000100:00080000:3.0:1775765333.047959:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765333.047962:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765333.047987:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765333.047997:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098793600/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765334.967566:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765337.015561:0:46654:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 2/2 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1775765337.015600:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 2 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775765337.015608:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026477b48/0x73cfd86cfc17c1eb lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x57b8ff0e03052cdb expref: -99 pid: 66222 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775765337.015709:0:46149:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x17:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775765337.015809:0:46149:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x17:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775765337.015897:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800028400000000 nid: local remote: 0x57b8ff0e03051b31 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1775765337.015991:0:46149:0:(vvp_io.c:1817:vvp_io_init()) [0x200003ab4:0x6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775765337.016024:0:46149:0:(vvp_io.c:288:vvp_io_fini()) [0x200003ab4:0x6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1775765337.016080:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026477b48/0x73cfd86cfc17c1eb lrc: 1/0,0 mode: --/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x57b8ff0e03052cdb expref: -99 pid: 66222 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775765337.016088:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804869400000020 nid: local remote: 0x57b8ff0e03051b31 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775765337.016094:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1775765337.016126:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b2389398000 lock: 0000000026477b48/0x73cfd86cfc17c1eb lrc: 0/0,0 mode: --/PR res: [0x280000400:0x11d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x57b8ff0e03052cdb expref: -99 pid: 66222 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775765337.016146:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b2389398000 lock: 00000000f63570e7/0x73cfd86cfc17bcf8 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x119:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804869400000020 nid: local remote: 0x57b8ff0e03051b31 expref: -99 pid: 61577 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775765337.016158:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765337.016160:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765337.016163:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765337.016173:0:2378:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:2.0:1775765337.016191:0:2378:0:(client.c:1830:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff8b23b9ea7800 x1862022098793728/t0(0) o103->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765338.167563:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765338.167576:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765338.167582:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765338.167594:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2799 00000100:00080000:3.0:1775765338.167607:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765338.167615:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765338.167675:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098793856/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765338.167709:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765338.167716:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765338.167718:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765338.167722:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2799 00000100:00080000:3.0:1775765338.167726:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765338.167729:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765338.167739:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765338.167753:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098793984/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765338.167766:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765338.167770:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765338.167771:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765338.167776:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2799 00000100:00080000:3.0:1775765338.167779:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765338.167783:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765338.167787:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765338.167802:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098794112/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765338.167814:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765338.167817:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765338.167819:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765338.167823:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2799 00000100:00080000:3.0:1775765338.167826:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765338.167829:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765338.167835:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765338.167848:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098794240/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765338.167858:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765338.167861:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765338.167863:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765338.167866:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2799 00000100:00080000:3.0:1775765338.167871:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765338.167875:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765338.167882:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765338.167905:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098794368/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765339.063647:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765341.111629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765343.159676:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765343.287586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765343.287596:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765343.287600:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765343.287610:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2804 00000100:00080000:3.0:1775765343.287622:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765343.287628:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765343.287667:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098794496/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765343.287699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765343.287703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765343.287705:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765343.287708:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2804 00000100:00080000:3.0:1775765343.287712:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765343.287716:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765343.287724:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765343.287736:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098794624/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765343.287746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765343.287749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765343.287750:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765343.287754:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2804 00000100:00080000:3.0:1775765343.287757:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765343.287760:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765343.287764:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765343.287772:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098794752/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765343.287782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765343.287785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765343.287786:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765343.287790:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2804 00000100:00080000:3.0:1775765343.287793:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765343.287796:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765343.287800:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765343.287812:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098794880/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765343.287834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765343.287839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765343.287841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765343.287845:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2804 00000100:00080000:3.0:1775765343.287847:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765343.287850:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765343.287856:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765343.287910:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098795008/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765345.207592:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765347.255540:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765348.407524:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765348.407532:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765348.407536:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765348.407544:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2809 00000100:00080000:3.0:1775765348.407554:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765348.407559:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765348.407591:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098795136/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765348.407619:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765348.407623:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765348.407625:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765348.407629:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2809 00000100:00080000:3.0:1775765348.407633:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765348.407637:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765348.407647:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765348.407670:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098795264/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765348.407683:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765348.407687:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765348.407689:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765348.407693:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2809 00000100:00080000:3.0:1775765348.407696:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765348.407699:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765348.407706:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765348.407718:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098795392/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765348.407725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765348.407728:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765348.407729:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765348.407732:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2809 00000100:00080000:3.0:1775765348.407735:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765348.407737:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765348.407740:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765348.407747:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098795520/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765348.407752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765348.407754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765348.407756:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765348.407758:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2809 00000100:00080000:3.0:1775765348.407760:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765348.407763:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765348.407766:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765348.407773:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098795648/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765349.303640:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765351.351613:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765353.399688:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765353.527675:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765353.527687:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765353.527692:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765353.527703:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2814 00000100:00080000:1.0:1775765353.527715:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765353.527722:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765353.527833:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098795776/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765353.527880:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765353.527887:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765353.527890:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765353.527896:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2814 00000100:00080000:1.0:1775765353.527903:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765353.527911:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765353.527923:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765353.527943:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098795904/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765353.527962:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765353.527966:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765353.527968:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765353.527974:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2814 00000100:00080000:1.0:1775765353.527978:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765353.527983:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765353.527988:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765353.528001:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098796032/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765353.528012:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765353.528016:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765353.528018:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765353.528023:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2814 00000100:00080000:1.0:1775765353.528049:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765353.528053:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765353.528057:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765353.528072:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098796160/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765353.528084:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765353.528088:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765353.528090:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765353.528097:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2814 00000100:00080000:1.0:1775765353.528101:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765353.528105:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765353.528111:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765353.528127:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098796288/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765355.447579:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765357.495577:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765358.647614:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765358.647625:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765358.647640:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765358.647661:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2820 00000100:00080000:3.0:1775765358.647674:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765358.647681:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765358.647745:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098796416/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765358.647788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765358.647795:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765358.647798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765358.647806:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2820 00000100:00080000:3.0:1775765358.647824:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765358.647833:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765358.647848:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765358.647868:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098796544/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765358.647898:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765358.647908:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765358.647910:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765358.647916:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2820 00000100:00080000:3.0:1775765358.647920:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765358.647927:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765358.647933:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765358.647946:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098796672/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765358.647959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765358.647963:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765358.647965:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765358.647970:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2820 00000100:00080000:3.0:1775765358.647973:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765358.647979:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765358.647985:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765358.648003:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098796800/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765358.648015:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765358.648019:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765358.648021:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765358.648028:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2820 00000100:00080000:3.0:1775765358.648033:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765358.648038:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765358.648045:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765358.648062:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098796928/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765359.543598:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765361.591647:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765363.639534:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765363.767560:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765363.767570:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765363.767576:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765363.767586:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2825 00000100:00080000:1.0:1775765363.767598:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765363.767606:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765363.767661:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098797056/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765363.767698:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765363.767703:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765363.767705:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765363.767710:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2825 00000100:00080000:1.0:1775765363.767714:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765363.767719:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765363.767738:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765363.767754:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098797184/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765363.767767:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765363.767769:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765363.767771:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765363.767774:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2825 00000100:00080000:1.0:1775765363.767777:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765363.767780:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765363.767784:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765363.767793:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098797312/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765363.767800:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765363.767802:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765363.767804:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765363.767806:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2825 00000100:00080000:1.0:1775765363.767809:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765363.767816:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765363.767820:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765363.767829:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098797440/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765363.767835:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765363.767838:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765363.767839:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765363.767841:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2825 00000100:00080000:1.0:1775765363.767851:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765363.767857:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765363.767863:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765363.767880:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098797568/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765365.687567:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765367.735734:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765368.887545:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765368.887557:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765368.887562:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765368.887585:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2830 00000100:00080000:1.0:1775765368.887599:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765368.887605:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765368.887657:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098797696/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765368.887679:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765368.887683:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765368.887684:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765368.887687:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2830 00000100:00080000:1.0:1775765368.887690:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765368.887693:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765368.887702:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765368.887713:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098797824/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765368.887726:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765368.887737:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765368.887739:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765368.887743:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2830 00000100:00080000:1.0:1775765368.887745:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765368.887748:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765368.887751:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765368.887761:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098797952/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765368.887769:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765368.887771:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765368.887773:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765368.887776:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2830 00000100:00080000:1.0:1775765368.887791:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765368.887794:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765368.887798:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765368.887808:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098798080/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765368.887815:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765368.887818:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765368.887819:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765368.887822:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2830 00000100:00080000:1.0:1775765368.887825:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765368.887827:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765368.887832:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765368.887843:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098798208/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765369.783606:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765371.831708:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765373.879697:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765374.007712:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765374.007728:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765374.007735:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765374.007750:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2835 00000100:00080000:1.0:1775765374.007783:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765374.007792:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765374.007848:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098798336/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765374.007908:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765374.007913:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765374.007916:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765374.007921:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2835 00000100:00080000:1.0:1775765374.007926:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765374.007931:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765374.007945:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765374.007965:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098798464/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765374.007983:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765374.007989:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765374.007996:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765374.008014:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2835 00000100:00080000:1.0:1775765374.008019:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765374.008025:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765374.008031:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765374.008045:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098798592/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765374.008091:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765374.008095:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765374.008096:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765374.008101:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2835 00000100:00080000:1.0:1775765374.008108:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765374.008120:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765374.008129:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765374.008141:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098798720/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765374.008151:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765374.008156:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765374.008158:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765374.008168:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2835 00000100:00080000:1.0:1775765374.008181:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765374.008195:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765374.008199:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765374.008215:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098798848/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765375.927559:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765377.975707:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765379.127680:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765379.127692:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765379.127698:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765379.127710:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2840 00000100:00080000:3.0:1775765379.127724:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765379.127732:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765379.127779:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098798976/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765379.127818:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765379.127823:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765379.127825:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765379.127830:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2840 00000100:00080000:3.0:1775765379.127834:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765379.127838:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765379.127849:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765379.127864:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098799104/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765379.127878:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765379.127886:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765379.127889:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765379.127895:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2840 00000100:00080000:3.0:1775765379.127899:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765379.127906:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765379.127911:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765379.127923:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098799232/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765379.127935:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765379.127939:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765379.127942:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765379.127946:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2840 00000100:00080000:3.0:1775765379.127950:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765379.127958:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765379.127965:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765379.127983:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098799360/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765379.127995:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765379.127999:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765379.128001:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765379.128005:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2840 00000100:00080000:3.0:1775765379.128010:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765379.128014:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765379.128021:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765379.128036:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098799488/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765380.023567:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765382.071671:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765384.119563:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765384.247585:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765384.247594:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765384.247599:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765384.247608:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2845 00000100:00080000:1.0:1775765384.247618:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765384.247624:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765384.247661:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098799616/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765384.247686:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765384.247690:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765384.247691:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765384.247694:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2845 00000100:00080000:1.0:1775765384.247697:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765384.247700:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765384.247707:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765384.247717:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098799744/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765384.247727:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765384.247730:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765384.247731:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765384.247734:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2845 00000100:00080000:1.0:1775765384.247738:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765384.247742:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765384.247747:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765384.247757:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098799872/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765384.247766:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765384.247770:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765384.247772:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765384.247784:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2845 00000100:00080000:1.0:1775765384.247792:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765384.247803:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765384.247810:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765384.247843:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098800000/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765384.247857:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765384.247860:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765384.247862:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765384.247867:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2845 00000100:00080000:1.0:1775765384.247871:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765384.247874:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765384.247880:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765384.247893:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098800128/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765386.167593:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765387.320010:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765384/real 1775765387] req@ffff8b23864ff800 x1862022098799616/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765387.320048:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098799616/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765387.320067:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765387.320075:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765387.320082:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098799616:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765387.320092:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765384/real 1775765387] req@ffff8b23864fc000 x1862022098799744/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765387.320109:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098799744/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765387.320118:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765387.320122:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765387.320126:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098799744:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765387.320171:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765384/real 1775765387] req@ffff8b23864fed80 x1862022098799872/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765387.320184:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098799872/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765387.320201:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765387.320206:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765387.320210:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098799872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765387.320217:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765384/real 1775765387] req@ffff8b23864fc380 x1862022098800000/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765387.320230:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098800000/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765387.320239:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765387.320243:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765387.320247:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098800000:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765387.320255:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765384/real 1775765387] req@ffff8b23864fdf80 x1862022098800128/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765387.320267:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098800128/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765387.320282:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765387.320286:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765387.320289:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098800128:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765388.215609:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765389.367612:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765389.367622:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765389.367627:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765389.367636:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2850 00000100:00080000:2.0:1775765389.367648:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765389.367657:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765389.367705:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022098800256/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765389.367747:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765389.367753:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765389.367755:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765389.367762:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2850 00000100:00080000:2.0:1775765389.367770:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765389.367777:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765389.367791:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765389.367809:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098800256:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765389.367827:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098800384/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765389.367848:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765389.367860:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765389.367863:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765389.367871:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2850 00000100:00080000:2.0:1775765389.367896:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765389.367904:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765389.367913:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765389.367922:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765389.367949:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098800512/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765389.367968:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765389.367973:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765389.367977:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765389.367983:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2850 00000100:00080000:2.0:1775765389.367990:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765389.367994:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765389.368003:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765389.368005:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765389/real 1775765389] req@ffff8b23b9ea6300 x1862022098800256/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765389.368034:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098800640/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765389.368055:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775765389.368066:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022098800256/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765389.368084:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765389.368088:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765389.368095:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765389.368105:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2850 00000100:00080000:3.0:1775765389.368109:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00080000:2.0:1775765389.368113:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775765389.368121:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098800256:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775765389.368127:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765389.368145:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765389.368167:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098800768/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765389.368182:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098800384:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765389.368222:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765389.368243:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765389/real 1775765389] req@ffff8b23b9ea7480 x1862022098800384/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765389.368258:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022098800384/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765389.368268:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765389.368272:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765389.368276:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098800384:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765389.368283:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098800512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765389.369194:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765389.369229:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765389/real 1775765389] req@ffff8b23b9ea4e00 x1862022098800512/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765389.369250:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022098800512/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765389.369268:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765389.369272:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765389.369276:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098800512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765389.369289:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098800640:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765389.369333:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765389.369347:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765389/real 1775765389] req@ffff8b23b9ea4380 x1862022098800640/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765389.369363:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022098800640/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765389.369372:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765389.369376:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765389.369380:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098800640:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765389.369410:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098800768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765389.369471:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765389.369484:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765389/real 1775765389] req@ffff8b23b9ea7100 x1862022098800768/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765389.369494:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022098800768/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765389.369501:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765389.369504:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765389.369507:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098800768:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765390.263632:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765392.311620:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765394.359680:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765394.487578:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765394.487586:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765394.487590:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765394.487598:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2855 00000100:00080000:1.0:1775765394.487607:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765394.487613:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765394.487647:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098800896/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765394.487672:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765394.487674:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765394.487676:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765394.487678:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2855 00000100:00080000:1.0:1775765394.487680:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765394.487682:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765394.487689:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765394.487698:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098801024/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765394.487706:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765394.487708:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765394.487709:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765394.487712:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2855 00000100:00080000:1.0:1775765394.487713:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765394.487715:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765394.487718:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765394.487728:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098801152/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765394.487739:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765394.487741:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765394.487743:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765394.487746:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2855 00000100:00080000:1.0:1775765394.487747:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765394.487749:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765394.487752:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765394.487761:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098801280/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765394.487763:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098800896:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775765394.487785:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765394.487788:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765394.487789:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765394.487792:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2855 00000100:00080000:1.0:1775765394.487794:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765394.487796:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765394.487800:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765394.487809:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098801408/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765394.487874:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765394.487924:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765394/real 1775765394] req@ffff8b23864fca80 x1862022098800896/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765394.487942:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098800896/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765394.487956:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765394.487962:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765394.487967:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098800896:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765394.487975:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098801024:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765394.488001:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765394.488015:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765394/real 1775765394] req@ffff8b23864fe680 x1862022098801024/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765394.488029:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098801024/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765394.488038:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765394.488042:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765394.488045:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098801024:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765394.488052:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098801152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765394.488076:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765394.488088:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765394/real 1775765394] req@ffff8b23864fce00 x1862022098801152/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765394.488101:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098801152/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765394.488107:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765394.488109:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765394.488111:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098801152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765394.488139:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098801280:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765394.488156:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765394.488163:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765394/real 1775765394] req@ffff8b23864fd180 x1862022098801280/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765394.488171:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098801280/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765394.488176:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765394.488178:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765394.488179:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098801280:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765394.488183:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098801408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765394.488194:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765394.488199:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765394/real 1775765394] req@ffff8b23864fea00 x1862022098801408/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765394.488207:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098801408/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765394.488212:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765394.488215:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765394.488217:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098801408:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765396.407563:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765398.455556:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765399.607574:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765399.607584:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765399.607589:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765399.607613:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2861 00000100:00080000:1.0:1775765399.607626:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765399.607633:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765399.607675:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098801536/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765399.607709:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765399.607713:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765399.607714:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765399.607718:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2861 00000100:00080000:1.0:1775765399.607721:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765399.607724:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765399.607740:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765399.607753:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098801664/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765399.607765:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765399.607768:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765399.607770:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765399.607774:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2861 00000100:00080000:1.0:1775765399.607777:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765399.607783:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765399.607788:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098801536:192.168.202.128@tcp:250:kworker.0 00000008:00100000:1.0:1775765399.607789:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765399.607826:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098801792/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765399.607838:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765399.607841:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765399.607843:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765399.607847:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2861 00000100:00080000:1.0:1775765399.607852:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765399.607855:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765399.607859:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765399.607866:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775765399.607890:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098801920/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765399.607907:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765399.607910:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765399.607914:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775765399.607915:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765399/real 1775765399] req@ffff8b23864fdc00 x1862022098801536/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765399.607921:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2861 00000100:00080000:1.0:1775765399.607928:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765399.607936:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765399.607944:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765399.607961:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098802048/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765399.607962:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098801536/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765399.607990:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765399.607996:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765399.608000:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098801536:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765399.608009:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098801664:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765399.608042:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765399.608060:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765399/real 1775765399] req@ffff8b23864fd500 x1862022098801664/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765399.608079:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098801664/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765399.608091:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765399.608096:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765399.608101:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098801664:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765399.608130:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098801792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765399.608152:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765399.608162:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765399/real 1775765399] req@ffff8b23864ff100 x1862022098801792/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765399.608173:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098801792/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765399.608180:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765399.608183:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765399.608186:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098801792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765399.608191:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098801920:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765399.608205:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765399.608212:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765399/real 1775765399] req@ffff8b23864fe300 x1862022098801920/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765399.608222:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098801920/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765399.608229:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765399.608231:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765399.608234:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098801920:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765399.608238:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098802048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765399.608251:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765399.608258:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765399/real 1775765399] req@ffff8b23864ffb80 x1862022098802048/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765399.608268:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098802048/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765399.608274:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765399.608277:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765399.608279:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098802048:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765400.503566:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765402.551554:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765404.599611:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765404.727526:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765404.727534:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765404.727538:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765404.727546:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2866 00000100:00080000:1.0:1775765404.727556:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765404.727561:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765404.727593:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098802176/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765404.727620:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765404.727623:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765404.727625:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765404.727627:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2866 00000100:00080000:1.0:1775765404.727630:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765404.727632:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765404.727640:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765404.727651:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098802304/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765404.727655:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098802176:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775765404.727665:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765404.727667:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765404.727669:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765404.727672:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2866 00000100:00080000:1.0:1775765404.727675:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765404.727677:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765404.727681:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765404.727690:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098802432/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765404.727696:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765404.727699:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765404.727700:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765404.727703:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2866 00000100:00080000:1.0:1775765404.727707:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765404.727710:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765404.727713:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765404.727720:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775765404.727722:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098802560/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765404.727729:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765404.727733:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765404.727735:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765404.727740:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2866 00000100:00080000:1.0:1775765404.727744:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765404.727748:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765404.727753:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765404.727769:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765404/real 1775765404] req@ffff8b23864ffb80 x1862022098802176/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765404.727772:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098802688/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765404.727800:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098802176/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765404.727809:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765404.727812:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765404.727816:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098802176:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765404.727838:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098802304:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765404.727856:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765404.727864:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765404/real 1775765404] req@ffff8b23864fe300 x1862022098802304/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765404.727872:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098802304/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765404.727882:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765404.727885:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765404.727887:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098802304:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765404.727890:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098802432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765404.727901:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765404.727907:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765404/real 1775765404] req@ffff8b23864ff100 x1862022098802432/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765404.727914:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098802432/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765404.727919:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765404.727921:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765404.727923:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098802432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765404.727926:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098802560:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765404.727945:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765404.727952:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765404/real 1775765404] req@ffff8b23864fd500 x1862022098802560/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765404.727959:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098802560/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765404.727964:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765404.727966:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765404.727969:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098802560:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765404.727975:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098802688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765404.727993:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765404.728003:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765404/real 1775765404] req@ffff8b23864fdc00 x1862022098802688/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765404.728016:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098802688/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765404.728025:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765404.728029:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765404.728032:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098802688:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765406.647569:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765408.695653:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765409.847538:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765409.847546:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765409.847550:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765409.847557:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2871 00000100:00080000:1.0:1775765409.847567:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765409.847574:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765409.847609:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098802816/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765409.847633:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765409.847637:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765409.847638:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765409.847641:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2871 00000100:00080000:1.0:1775765409.847644:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765409.847647:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765409.847655:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765409.847664:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098802816:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775765409.847667:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098802944/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765409.847679:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765409.847681:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765409.847682:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765409.847686:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2871 00000100:00080000:1.0:1775765409.847689:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765409.847693:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765409.847698:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765409.847711:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098803072/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765409.847721:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765409.847724:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765409.847727:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775765409.847729:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775765409.847736:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2871 00000100:00080000:1.0:1775765409.847742:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765409.847747:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765409.847754:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765409.847767:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098803200/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765409.847770:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765409/real 1775765409] req@ffff8b23864fea00 x1862022098802816/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765409.847790:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765409.847793:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765409.847794:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765409.847798:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2871 00000100:00100000:3.0:1775765409.847802:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098802816/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765409.847803:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765409.847810:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765409.847818:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775765409.847821:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765409.847827:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:1.0:1775765409.847829:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098803328/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765409.847834:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098802816:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765409.847865:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098802944:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765409.847901:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765409.847915:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765409/real 1775765409] req@ffff8b23864fd180 x1862022098802944/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765409.847929:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098802944/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765409.847936:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765409.847938:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765409.847941:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098802944:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765409.847945:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098803072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765409.847961:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765409.847968:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765409/real 1775765409] req@ffff8b23864fce00 x1862022098803072/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765409.847976:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098803072/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765409.847981:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765409.847983:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765409.847985:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098803072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765409.847989:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098803200:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765409.847999:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765409.848004:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765409/real 1775765409] req@ffff8b23864fe680 x1862022098803200/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765409.848011:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098803200/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765409.848026:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765409.848028:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765409.848030:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098803200:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765409.848034:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098803328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765409.848045:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765409.848051:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765409/real 1775765409] req@ffff8b23864fca80 x1862022098803328/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765409.848058:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098803328/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765409.848063:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765409.848065:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765409.848066:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098803328:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765410.743574:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765412.791582:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765414.839571:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765414.967554:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765414.967562:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765414.967566:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765414.967573:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2876 00000100:00080000:3.0:1775765414.967582:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765414.967587:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765414.967619:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098803456/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765414.967643:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765414.967647:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765414.967648:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765414.967651:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2876 00000100:00080000:3.0:1775765414.967653:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765414.967655:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765414.967662:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765414.967670:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098803584/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765414.967678:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765414.967681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765414.967682:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765414.967685:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2876 00000100:00080000:3.0:1775765414.967687:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765414.967690:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765414.967694:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765414.967700:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098803712/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765414.967714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765414.967717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765414.967718:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765414.967721:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2876 00000100:00080000:3.0:1775765414.967723:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765414.967725:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765414.967728:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765414.967736:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098803840/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765414.967743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765414.967745:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765414.967746:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765414.967749:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2876 00000100:00080000:3.0:1775765414.967751:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765414.967753:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765414.967756:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765414.967763:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098803968/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765416.887638:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765418.935563:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765420.087514:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765420.087522:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765420.087526:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765420.087534:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2881 00000100:00080000:3.0:1775765420.087544:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765420.087549:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765420.087581:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098804096/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765420.087603:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765420.087605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765420.087607:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765420.087609:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2881 00000100:00080000:3.0:1775765420.087612:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765420.087615:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765420.087622:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765420.087632:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098804224/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765420.087641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765420.087643:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765420.087645:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765420.087650:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2881 00000100:00080000:3.0:1775765420.087655:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765420.087660:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765420.087663:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765420.087671:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098804352/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765420.087677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765420.087679:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765420.087681:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765420.087683:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2881 00000100:00080000:3.0:1775765420.087685:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765420.087687:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765420.087690:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765420.087696:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098804480/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765420.087706:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765420.087708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765420.087710:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765420.087714:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2881 00000100:00080000:3.0:1775765420.087717:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765420.087721:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765420.087724:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765420.087731:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098804608/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765420.983568:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765423.031557:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765425.079572:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765425.207503:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765425.207511:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765425.207516:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765425.207524:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2886 00000100:00080000:3.0:1775765425.207532:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765425.207538:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765425.207575:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098804736/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765425.207604:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765425.207607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765425.207609:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765425.207612:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2886 00000100:00080000:3.0:1775765425.207615:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765425.207618:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765425.207626:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765425.207637:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098804864/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765425.207645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765425.207648:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765425.207649:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765425.207653:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2886 00000100:00080000:3.0:1775765425.207657:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765425.207662:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765425.207666:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765425.207680:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098804992/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765425.207697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765425.207701:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765425.207703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765425.207707:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2886 00000100:00080000:3.0:1775765425.207711:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765425.207714:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765425.207719:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765425.207729:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098805120/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765425.207735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765425.207738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765425.207739:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765425.207742:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2886 00000100:00080000:3.0:1775765425.207744:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765425.207749:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765425.207753:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765425.207769:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098805248/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765427.127588:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765429.175678:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765430.327644:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765430.327656:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765430.327662:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765430.327673:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2891 00000100:00080000:1.0:1775765430.327686:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765430.327701:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765430.327736:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098805376/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765430.327760:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765430.327781:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765430.327783:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765430.327788:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2891 00000100:00080000:1.0:1775765430.327794:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765430.327799:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765430.327810:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765430.327830:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098805504/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765430.327848:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765430.327852:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765430.327854:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765430.327866:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2891 00000100:00080000:1.0:1775765430.327874:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765430.327881:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765430.327887:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765430.327906:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098805632/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765430.327918:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765430.327922:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765430.327924:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765430.327928:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2891 00000100:00080000:1.0:1775765430.327932:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765430.327936:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765430.327941:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765430.327952:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098805760/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765430.327983:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765430.327987:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765430.327990:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765430.327996:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2891 00000100:00080000:1.0:1775765430.328011:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765430.328016:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765430.328021:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765430.328038:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098805888/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765431.223575:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765433.271697:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765435.319565:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765435.447549:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765435.447565:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765435.447570:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765435.447580:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2896 00000100:00080000:3.0:1775765435.447592:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765435.447599:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765435.447637:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098806016/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765435.447662:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765435.447665:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765435.447667:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765435.447670:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2896 00000100:00080000:3.0:1775765435.447673:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765435.447675:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765435.447682:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765435.447691:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098806144/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765435.447704:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765435.447707:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765435.447708:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765435.447711:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2896 00000100:00080000:3.0:1775765435.447713:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765435.447716:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765435.447719:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765435.447729:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098806272/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765435.447736:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765435.447738:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765435.447739:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765435.447742:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2896 00000100:00080000:3.0:1775765435.447744:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765435.447746:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765435.447749:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765435.447756:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098806400/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765435.447762:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765435.447764:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765435.447765:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765435.447767:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2896 00000100:00080000:3.0:1775765435.447771:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765435.447773:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765435.447776:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765435.447785:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098806528/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765437.367624:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765439.415561:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765440.567573:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765440.567581:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765440.567585:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765440.567593:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2901 00000100:00080000:2.0:1775765440.567602:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765440.567607:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765440.567659:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098806656/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765440.567681:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765440.567684:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765440.567685:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765440.567687:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2901 00000100:00080000:2.0:1775765440.567690:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765440.567693:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765440.567701:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765440.567709:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098806784/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765440.567718:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765440.567720:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765440.567722:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765440.567725:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2901 00000100:00080000:2.0:1775765440.567727:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765440.567730:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765440.567738:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765440.567749:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098806912/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765440.567756:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765440.567759:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765440.567760:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765440.567763:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2901 00000100:00080000:2.0:1775765440.567765:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765440.567768:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765440.567771:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765440.567778:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098807040/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765440.567784:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765440.567787:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765440.567788:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765440.567790:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2901 00000100:00080000:2.0:1775765440.567793:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765440.567798:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765440.567804:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765440.567811:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098807168/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765441.463548:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765443.511639:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765445.559615:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765445.687599:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765445.687613:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765445.687619:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765445.687630:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2907 00000100:00080000:3.0:1775765445.687643:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765445.687650:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765445.687691:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098807296/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765445.687726:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765445.687730:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765445.687732:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765445.687736:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2907 00000100:00080000:3.0:1775765445.687739:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765445.687742:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765445.687752:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765445.687765:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098807424/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765445.687777:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765445.687783:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765445.687785:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765445.687797:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2907 00000100:00080000:3.0:1775765445.687800:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765445.687803:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765445.687808:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765445.687818:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098807552/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765445.687827:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765445.687830:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765445.687831:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765445.687835:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2907 00000100:00080000:3.0:1775765445.687839:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765445.687842:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765445.687847:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765445.687860:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098807680/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765445.687873:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765445.687876:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765445.687878:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765445.687881:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2907 00000100:00080000:3.0:1775765445.687884:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765445.687887:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765445.687892:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765445.687928:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098807808/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765447.607555:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765449.655611:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765450.807573:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765450.807600:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765450.807607:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765450.807618:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2912 00000100:00080000:2.0:1775765450.807633:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765450.807653:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765450.807708:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098807936/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765450.807734:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765450.807738:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765450.807740:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765450.807745:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2912 00000100:00080000:2.0:1775765450.807750:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765450.807754:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765450.807765:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765450.807780:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098808064/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765450.807795:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765450.807799:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765450.807801:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765450.807807:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2912 00000100:00080000:2.0:1775765450.807811:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765450.807816:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765450.807821:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765450.807833:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022098808192/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765450.807842:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765450.807844:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765450.807846:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765450.807875:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2912 00000100:00080000:2.0:1775765450.807881:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765450.807884:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765450.807889:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765450.807901:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098808320/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765450.807915:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765450.807918:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765450.807919:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765450.807922:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2912 00000100:00080000:2.0:1775765450.807925:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765450.807927:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765450.807932:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765450.807944:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022098808448/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765451.703659:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765453.751614:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775765453.879729:0:2364:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775765453.879769:0:2364:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775765453.879787:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765453.879840:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765453.879850:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765453.879858:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765453.879866:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00010000:00010000:2.0:1775765455.799570:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765455.927604:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765455.927612:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765455.927616:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765455.927625:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2917 00000100:00080000:3.0:1775765455.927634:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765455.927640:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765455.927673:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098808576/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765455.927693:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765455.927696:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765455.927698:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765455.927700:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2917 00000100:00080000:3.0:1775765455.927703:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765455.927706:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765455.927714:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765455.927724:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098808704/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765455.927739:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765455.927741:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765455.927743:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765455.927746:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2917 00000100:00080000:3.0:1775765455.927749:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765455.927752:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765455.927755:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765455.927777:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098808832/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765455.927786:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765455.927790:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765455.927792:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765455.927796:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2917 00000100:00080000:3.0:1775765455.927811:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765455.927818:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765455.927825:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765455.927851:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098808960/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765455.927861:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765455.927864:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765455.927867:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765455.927871:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2917 00000100:00080000:3.0:1775765455.927875:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765455.927882:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765455.927888:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765455.927907:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098809088/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765457.847605:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765459.895620:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765461.047535:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765461.047552:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765461.047556:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765461.047566:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2922 00000100:00080000:3.0:1775765461.047577:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765461.047583:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765461.047619:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098809216/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765461.047650:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765461.047664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765461.047666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765461.047669:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2922 00000100:00080000:3.0:1775765461.047672:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765461.047675:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765461.047683:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765461.047693:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098809344/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765461.047702:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765461.047705:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765461.047707:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765461.047710:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2922 00000100:00080000:3.0:1775765461.047712:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765461.047715:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765461.047719:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765461.047726:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098809472/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765461.047733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765461.047736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765461.047737:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765461.047741:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2922 00000100:00080000:3.0:1775765461.047744:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765461.047747:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765461.047804:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765461.047823:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098809600/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765461.047838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765461.047843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765461.047845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765461.047859:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2922 00000100:00080000:3.0:1775765461.047864:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765461.047868:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765461.047874:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765461.047885:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098809728/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765461.943611:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765463.991573:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765466.039638:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765466.167803:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098809856:192.168.202.128@tcp:250:kworker.0 00000800:00000100:3.0:1775765466.167875:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765466.167936:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765466/real 1775765466] req@ffff8b2382dae680 x1862022098809856/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765466.167973:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098809856/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765466.167993:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765466.168001:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765466.168007:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098809856:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765466.168039:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098809984:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765466.168070:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765466.168079:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765466/real 1775765466] req@ffff8b2382dad500 x1862022098809984/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765466.168089:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098809984/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765466.168096:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765466.168099:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765466.168101:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098809984:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765466.168106:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098810112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765466.168119:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765466.168126:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765466/real 1775765466] req@ffff8b2382daea00 x1862022098810112/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765466.168135:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098810112/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765466.168148:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765466.168153:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765466.168157:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098810112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765466.168164:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098810240:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765466.168190:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765466.168203:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765466/real 1775765466] req@ffff8b2382daed80 x1862022098810240/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765466.168220:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098810240/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765466.168230:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765466.168235:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765466.168238:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098810240:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765466.168244:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098810368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765466.168263:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765466.168272:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765466/real 1775765466] req@ffff8b2382dace00 x1862022098810368/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765466.168281:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098810368/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765466.168287:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765466.168290:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765466.168293:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098810368:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765468.087623:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765470.135736:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765471.287570:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765471.287578:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765471.287582:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765471.287590:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2932 00000100:00080000:1.0:1775765471.287600:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765471.287606:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765471.287677:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098810496/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765471.287711:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765471.287714:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765471.287715:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765471.287718:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2932 00000100:00080000:1.0:1775765471.287720:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765471.287723:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765471.287731:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765471.287742:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098810624/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765471.287752:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765471.287754:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765471.287755:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765471.287761:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2932 00000100:00080000:1.0:1775765471.287763:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775765471.287763:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098810496:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775765471.287770:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765471.287774:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765471.287784:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098810752/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765471.287791:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765471.287794:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765471.287795:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765471.287798:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2932 00000100:00080000:1.0:1775765471.287801:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765471.287803:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765471.287806:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765471.287813:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098810880/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765471.287819:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765471.287821:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765471.287823:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775765471.287825:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775765471.287829:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2932 00000100:00080000:1.0:1775765471.287832:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765471.287835:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765471.287838:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765471.287846:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098811008/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765471.287872:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765471/real 1775765471] req@ffff8b23864ff800 x1862022098810496/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765471.287889:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098810496/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765471.287897:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765471.287901:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765471.287904:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098810496:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765471.287909:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098810624:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765471.287924:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765471.287932:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765471/real 1775765471] req@ffff8b23864fc000 x1862022098810624/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765471.287940:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098810624/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765471.287945:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765471.287947:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765471.287949:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098810624:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765471.287966:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098810752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765471.287983:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765471.287994:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765471/real 1775765471] req@ffff8b23864fed80 x1862022098810752/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765471.288006:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098810752/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765471.288015:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765471.288019:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765471.288022:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098810752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765471.288029:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098810880:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765471.288050:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765471.288059:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765471/real 1775765471] req@ffff8b23864fc380 x1862022098810880/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765471.288067:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098810880/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765471.288073:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765471.288075:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765471.288077:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098810880:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765471.288081:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098811008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765471.288092:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765471.288098:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765471/real 1775765471] req@ffff8b23864fdf80 x1862022098811008/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765471.288104:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098811008/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765471.288110:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765471.288112:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765471.288114:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098811008:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765472.183668:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765474.231763:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765476.279666:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765476.407559:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765476.407575:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765476.407582:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765476.407598:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2937 00000100:00080000:1.0:1775765476.407640:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765476.407653:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765476.407713:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098811136/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765476.407760:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765476.407765:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765476.407768:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765476.407774:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2937 00000100:00080000:1.0:1775765476.407780:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765476.407785:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765476.407799:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765476.407816:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098811264/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765476.407840:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765476.407845:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765476.407859:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765476.407864:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2937 00000100:00080000:1.0:1775765476.407868:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765476.407877:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765476.407882:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098811136:192.168.202.128@tcp:250:kworker.0 00000008:00100000:1.0:1775765476.407883:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765476.407902:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098811392/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765476.407917:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765476.407921:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765476.407924:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765476.407928:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2937 00000100:00080000:1.0:1775765476.407933:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765476.407937:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765476.407944:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765476.407959:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098811520/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765476.407971:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1775765476.407981:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775765476.407983:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765476.407988:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765476.407994:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2937 00000100:00080000:1.0:1775765476.408002:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765476.408007:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765476.408014:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765476.408034:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098811648/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765476.408068:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765476/real 1775765476] req@ffff8b23864fca80 x1862022098811136/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765476.408098:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098811136/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765476.408112:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765476.408121:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765476.408127:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098811136:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765476.408137:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098811264:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765476.408165:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765476.408180:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765476/real 1775765476] req@ffff8b23864fe680 x1862022098811264/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765476.408194:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098811264/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765476.408204:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765476.408218:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765476.408222:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098811264:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765476.408256:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098811392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765476.408276:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765476.408295:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765476/real 1775765476] req@ffff8b23864fce00 x1862022098811392/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765476.408309:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098811392/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765476.408318:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765476.408321:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765476.408325:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098811392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765476.408332:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098811520:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765476.408361:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765476.408370:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765476/real 1775765476] req@ffff8b23864fd180 x1862022098811520/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765476.408384:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098811520/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765476.408393:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765476.408396:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765476.408400:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098811520:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765476.408406:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098811648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765476.408432:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765476.408441:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765476/real 1775765476] req@ffff8b23864fea00 x1862022098811648/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765476.408515:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098811648/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765476.408526:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765476.408530:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765476.408534:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098811648:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765478.327578:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765480.375697:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765481.527933:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098811776:192.168.202.128@tcp:250:kworker.0 00000800:00000100:3.0:1775765481.528070:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765481.528160:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765481/real 1775765481] req@ffff8b2382dafb80 x1862022098811776/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765481.528202:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098811776/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765481.528220:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765481.528243:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765481.528249:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098811776:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765481.528286:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098811904:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765481.528317:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765481.528332:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765481/real 1775765481] req@ffff8b2382dac700 x1862022098811904/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765481.528346:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098811904/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765481.528374:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765481.528378:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765481.528383:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098811904:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765481.528389:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098812032:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765481.528407:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765481.528417:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765481/real 1775765481] req@ffff8b2382daf480 x1862022098812032/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765481.528429:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098812032/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765481.528439:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765481.528442:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765481.528445:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098812032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765481.528451:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098812160:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765481.528523:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765481.528544:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765481/real 1775765481] req@ffff8b2382dadf80 x1862022098812160/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765481.528591:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098812160/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765481.528607:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765481.528613:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765481.528621:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098812160:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765481.528633:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098812288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765481.528670:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765481.528685:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765481/real 1775765481] req@ffff8b2382dace00 x1862022098812288/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765481.528699:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098812288/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765481.528709:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765481.528713:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765481.528717:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098812288:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765482.423602:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765484.471588:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765486.519738:0:46654:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 3/3 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1775765486.519804:0:46149:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 3 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775765486.519813:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17c3a4 lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e030532e6 expref: -99 pid: 68217 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775765486.519899:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e800/0x73cfd86cfc17c39d lrc: 2/0,0 mode: PR/PR res: [0x200000002:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x57b8ff0e030532ca expref: -99 pid: 68217 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775765486.519917:0:46149:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17c38f lrc: 2/0,0 mode: PR/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x57b8ff0e03053299 expref: -99 pid: 67476 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1775765486.519970:0:46149:0:(vvp_object.c:117:vvp_conf_set()) [0x200003ab4:0x1f:0x0]: losing layout lock 00010000:00010000:2.0:1775765486.520065:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17c3a4 lrc: 1/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030532e6 expref: -99 pid: 68217 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775765486.520097:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e800/0x73cfd86cfc17c39d lrc: 1/0,0 mode: --/PR res: [0x200000002:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030532ca expref: -99 pid: 68217 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775765486.520112:0:46149:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17c38f lrc: 1/0,0 mode: --/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03053299 expref: -99 pid: 67476 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775765486.520120:0:46149:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1775765486.520167:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5fc00/0x73cfd86cfc17c3a4 lrc: 0/0,0 mode: --/PR res: [0x200000002:0x2:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030532e6 expref: -99 pid: 68217 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775765486.520191:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b9c5e800/0x73cfd86cfc17c39d lrc: 0/0,0 mode: --/PR res: [0x200000002:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x57b8ff0e030532ca expref: -99 pid: 68217 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775765486.520212:0:46149:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b2389398000 lock: ffff8b23b8740000/0x73cfd86cfc17c38f lrc: 0/0,0 mode: --/PR res: [0x200003ab4:0x1f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x57b8ff0e03053299 expref: -99 pid: 67476 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1775765486.520244:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765486.520246:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765486.520250:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775765486.520263:0:2379:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00080000:2.0:1775765486.520290:0:2379:0:(client.c:1830:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff8b23b9ea5180 x1862022098812416/t0(0) o103->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:17/18 lens 336/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765486.647655:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765486.647670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765486.647677:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765486.647693:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2948 00000100:00080000:3.0:1775765486.647711:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765486.647724:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765486.647782:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098812544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765486.647819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765486.647824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765486.647826:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765486.647843:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2948 00000100:00080000:3.0:1775765486.647850:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765486.647858:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765486.647869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765486.647891:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098812672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765486.647918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765486.647923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765486.647925:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765486.647929:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2948 00000100:00080000:3.0:1775765486.647935:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765486.647939:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765486.647945:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765486.647981:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098812800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765486.647995:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765486.647999:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765486.648001:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765486.648006:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2948 00000100:00080000:3.0:1775765486.648010:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765486.648016:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765486.648023:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765486.648044:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098812928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765486.648062:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765486.648068:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765486.648070:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765486.648075:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2948 00000100:00080000:3.0:1775765486.648078:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765486.648086:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765486.648098:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765486.648127:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098813056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765488.567687:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765490.615727:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765491.767583:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765491.767594:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765491.767600:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765491.767611:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2953 00000100:00080000:3.0:1775765491.767631:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765491.767639:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765491.767686:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098813184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765491.767715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765491.767718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765491.767720:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765491.767723:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2953 00000100:00080000:3.0:1775765491.767726:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765491.767728:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765491.767738:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765491.767750:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098813312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765491.767761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765491.767764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765491.767765:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765491.767771:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2953 00000100:00080000:3.0:1775765491.767774:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765491.767778:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765491.767785:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765491.767793:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098813440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765491.767800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765491.767803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765491.767804:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765491.767807:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2953 00000100:00080000:3.0:1775765491.767809:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765491.767811:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765491.767814:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765491.767840:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098813568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765491.767846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765491.767849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765491.767850:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765491.767863:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2953 00000100:00080000:3.0:1775765491.767867:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765491.767870:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765491.767874:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765491.767899:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098813696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765492.663672:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765494.711604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765496.759562:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765496.887758:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098813824:192.168.202.128@tcp:250:kworker.0 00000800:00000100:3.0:1775765496.887855:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765496.887925:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765496/real 1775765496] req@ffff8b2382daed80 x1862022098813824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765496.887963:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098813824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765496.888007:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765496.888021:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765496.888027:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098813824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765496.888062:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098813952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765496.888089:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765496.888101:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765496/real 1775765496] req@ffff8b2382daea00 x1862022098813952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765496.888111:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098813952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765496.888118:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765496.888121:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765496.888125:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098813952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765496.888130:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098814080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765496.888144:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765496.888151:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765496/real 1775765496] req@ffff8b2382dad500 x1862022098814080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765496.888161:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098814080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765496.888167:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765496.888169:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765496.888171:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098814080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765496.888176:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098814208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765496.888189:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765496.888212:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765496/real 1775765496] req@ffff8b2382dae680 x1862022098814208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765496.888223:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098814208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765496.888239:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765496.888242:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765496.888245:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098814208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765496.888250:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098814336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765496.888265:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765496.888273:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765496/real 1775765496] req@ffff8b2382dac380 x1862022098814336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765496.888283:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098814336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765496.888306:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765496.888310:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765496.888313:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098814336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765498.807651:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765500.855644:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765502.007551:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765502.007558:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765502.007561:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765502.007569:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2963 00000100:00080000:2.0:1775765502.007578:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765502.007584:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765502.007616:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022098814464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765502.007638:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765502.007641:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765502.007643:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765502.007645:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2963 00000100:00080000:2.0:1775765502.007647:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765502.007650:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765502.007657:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765502.007665:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098814592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765502.007673:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765502.007675:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765502.007676:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765502.007679:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2963 00000100:00080000:2.0:1775765502.007681:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765502.007683:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765502.007686:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765502.007693:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098814720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765502.007700:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765502.007703:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765502.007704:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765502.007709:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2963 00000100:00100000:3.0:1775765502.007709:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098814464:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775765502.007714:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765502.007718:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765502.007721:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765502.007729:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022098814848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765502.007738:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765502.007741:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765502.007743:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765502.007746:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2963 00000100:00080000:2.0:1775765502.007760:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765502.007764:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775765502.007770:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:2.0:1775765502.007771:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765502.007787:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098814976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765502.007814:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765502/real 1775765502] req@ffff8b23b9ea6d80 x1862022098814464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765502.007829:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022098814464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765502.007838:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765502.007841:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765502.007844:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098814464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765502.007849:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098814592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765502.007864:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765502.007871:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765502/real 1775765502] req@ffff8b23b9ea5f80 x1862022098814592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765502.007879:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022098814592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765502.007888:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765502.007890:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765502.007892:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098814592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765502.007910:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098814720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765502.007920:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765502.007925:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765502/real 1775765502] req@ffff8b23b9ea6680 x1862022098814720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765502.007932:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022098814720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765502.007937:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765502.007939:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765502.007941:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098814720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765502.007944:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098814848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765502.007954:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765502.007959:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765502/real 1775765502] req@ffff8b23b9ea5880 x1862022098814848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765502.007967:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022098814848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765502.007971:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765502.007973:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765502.007975:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098814848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765502.007979:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098814976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765502.007988:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765502.007993:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765502/real 1775765502] req@ffff8b23b9ea5c00 x1862022098814976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765502.007999:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022098814976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765502.008004:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765502.008006:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765502.008007:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098814976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765502.903643:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765504.951551:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765506.999625:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765507.127536:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765507.127550:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765507.127556:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765507.127565:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2968 00000100:00080000:2.0:1775765507.127576:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765507.127583:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765507.127631:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022098815104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765507.127670:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765507.127676:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765507.127679:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765507.127695:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2968 00000100:00080000:2.0:1775765507.127701:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765507.127706:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765507.127718:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765507.127733:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098815232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765507.127746:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765507.127749:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765507.127752:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765507.127756:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2968 00000100:00080000:2.0:1775765507.127760:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775765507.127766:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098815104:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775765507.127767:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765507.127774:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765507.127785:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098815360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765507.127795:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765507.127799:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765507.127800:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765507.127804:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2968 00000100:00080000:2.0:1775765507.127807:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765507.127811:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765507.127816:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765507.127826:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098815488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765507.127835:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765507.127838:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765507.127840:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765507.127843:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2968 00000100:00080000:2.0:1775765507.127846:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765507.127850:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775765507.127857:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:2.0:1775765507.127858:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765507.127872:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098815616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765507.127912:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765507/real 1775765507] req@ffff8b23b9ea7b80 x1862022098815104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765507.127936:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022098815104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765507.127951:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765507.127957:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765507.127963:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098815104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765507.127973:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098815232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765507.128006:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765507.128024:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765507/real 1775765507] req@ffff8b23b9ea6a00 x1862022098815232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765507.128036:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022098815232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765507.128044:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765507.128047:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765507.128050:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098815232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765507.128096:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098815360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765507.128117:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765507.128127:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765507/real 1775765507] req@ffff8b23b9ea4000 x1862022098815360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765507.128139:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022098815360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765507.128146:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765507.128150:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765507.128153:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098815360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765507.128158:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098815488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765507.128173:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765507.128181:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765507/real 1775765507] req@ffff8b23b9ea5500 x1862022098815488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765507.128191:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022098815488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765507.128200:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765507.128203:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765507.128206:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098815488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765507.128211:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098815616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765507.128225:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765507.128232:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765507/real 1775765507] req@ffff8b23b9ea4700 x1862022098815616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765507.128242:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022098815616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765507.128249:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765507.128252:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765507.128255:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098815616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765509.047622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765511.095573:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765512.247606:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765512.247616:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765512.247621:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765512.247633:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2973 00000100:00080000:1.0:1775765512.247645:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765512.247652:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765512.247691:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098815744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765512.247722:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765512.247726:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765512.247728:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765512.247732:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2973 00000100:00080000:1.0:1775765512.247734:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765512.247737:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765512.247745:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765512.247755:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098815872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765512.247768:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765512.247774:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765512.247777:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765512.247781:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2973 00000100:00080000:1.0:1775765512.247783:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765512.247788:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765512.247793:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765512.247812:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098816000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765512.247821:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098815744:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775765512.247825:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765512.247841:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765512.247846:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765512.247851:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2973 00000100:00080000:1.0:1775765512.247854:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765512.247857:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765512.247861:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765512.247874:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098816128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765512.247886:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765512.247891:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765512.247894:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765512.247900:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2973 00000100:00080000:1.0:1775765512.247907:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765512.247916:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765512.247924:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765512.247947:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098816256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765512.247964:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765512.248054:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765512/real 1775765512] req@ffff8b23864fdc00 x1862022098815744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765512.248086:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098815744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765512.248102:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765512.248109:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765512.248115:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098815744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765512.248127:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098815872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765512.248156:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765512.248172:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765512/real 1775765512] req@ffff8b23864fd500 x1862022098815872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765512.248187:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098815872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765512.248198:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765512.248202:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765512.248206:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098815872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765512.248243:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098816000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765512.248299:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765512.248321:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765512/real 1775765512] req@ffff8b23864ff100 x1862022098816000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765512.248344:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098816000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765512.248362:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765512.248368:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765512.248374:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098816000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765512.248387:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098816128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765512.248433:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765512.248519:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765512/real 1775765512] req@ffff8b23864fe300 x1862022098816128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765512.248550:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098816128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765512.248568:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765512.248575:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765512.248582:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098816128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765512.248596:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098816256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765512.248632:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765512.248647:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765512/real 1775765512] req@ffff8b23864ffb80 x1862022098816256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765512.248662:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098816256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765512.248673:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765512.248677:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765512.248681:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098816256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765513.143610:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765515.191647:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765517.239572:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765517.367571:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765517.367581:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765517.367586:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765517.367598:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2978 00000100:00080000:3.0:1775765517.367613:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765517.367621:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765517.367690:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098816384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765517.367722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765517.367725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765517.367727:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765517.367731:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2978 00000100:00080000:3.0:1775765517.367735:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765517.367739:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765517.367749:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765517.367761:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098816512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765517.367773:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765517.367779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765517.367785:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765517.367793:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2978 00000100:00080000:3.0:1775765517.367798:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765517.367803:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765517.367810:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765517.367830:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098816640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765517.367845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765517.367850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765517.367852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765517.367858:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2978 00000100:00080000:3.0:1775765517.367865:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765517.367870:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765517.367878:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765517.367895:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098816768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765517.367910:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765517.367915:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765517.367918:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765517.367929:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2978 00000100:00080000:3.0:1775765517.367937:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765517.367943:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765517.367951:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765517.367991:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098816896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765519.287598:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775765520.439860:0:2365:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775765520.439908:0:2365:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775765520.439927:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775765520.439969:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775765520.439983:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775765520.439996:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775765520.440022:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00010000:00010000:2.0:1775765521.335577:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765522.487600:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765522.487609:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765522.487613:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765522.487622:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2983 00000100:00080000:1.0:1775765522.487633:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765522.487640:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765522.487685:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098817024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765522.487712:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765522.487716:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765522.487717:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765522.487720:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2983 00000100:00080000:1.0:1775765522.487723:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765522.487726:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765522.487735:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765522.487749:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098817152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765522.487766:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765522.487770:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765522.487775:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765522.487784:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2983 00000100:00080000:1.0:1775765522.487793:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765522.487799:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765522.487807:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765522.487827:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098817280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765522.487840:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765522.487845:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765522.487847:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765522.487853:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2983 00000100:00080000:1.0:1775765522.487862:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765522.487868:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765522.487875:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765522.487894:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098817408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765522.487907:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765522.487912:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765522.487915:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765522.487921:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2983 00000100:00080000:1.0:1775765522.487927:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765522.487936:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765522.487948:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765522.487974:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098817536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765523.383575:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765525.431586:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765527.481633:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765527.607983:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098817664:192.168.202.128@tcp:250:kworker.0 00000800:00000100:3.0:1775765527.608163:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765527.608245:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765527/real 1775765527] req@ffff8b2382dac000 x1862022098817664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765527.608273:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098817664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765527.608287:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765527.608293:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765527.608298:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098817664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765527.608309:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098817792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765527.608343:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765527.608359:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765527/real 1775765527] req@ffff8b2382daf100 x1862022098817792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765527.608387:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098817792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765527.608415:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765527.608419:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765527.608424:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098817792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765527.608518:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098817920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765527.608573:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765527.608587:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765527/real 1775765527] req@ffff8b2382daca80 x1862022098817920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765527.608600:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098817920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765527.608611:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765527.608614:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765527.608618:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098817920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765527.608626:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098818048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765527.608644:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765527.608654:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765527/real 1775765527] req@ffff8b2382dad880 x1862022098818048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765527.608667:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098818048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765527.608675:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765527.608691:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765527.608700:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098818048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765527.608707:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098818176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765527.608726:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765527.608736:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765527/real 1775765527] req@ffff8b2382dac380 x1862022098818176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765527.608748:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098818176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765527.608757:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765527.608760:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765527.608763:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098818176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765529.527594:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765531.575609:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765532.727577:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765532.727586:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765532.727590:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765532.727599:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2994 00000100:00080000:1.0:1775765532.727609:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765532.727616:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765532.727664:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098818304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765532.727692:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765532.727697:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765532.727699:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765532.727704:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2994 00000100:00080000:1.0:1775765532.727709:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765532.727713:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765532.727725:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765532.727741:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098818432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765532.727766:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765532.727771:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765532.727774:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765532.727778:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2994 00000100:00080000:1.0:1775765532.727782:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765532.727786:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765532.727793:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765532.727794:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098818304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775765532.727807:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098818560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765532.727816:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765532.727819:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765532.727820:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765532.727824:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2994 00000100:00080000:1.0:1775765532.727826:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765532.727828:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765532.727831:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765532.727838:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098818688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765532.727857:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765532.727860:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765532.727861:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765532.727864:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2994 00000100:00080000:1.0:1775765532.727867:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765532.727869:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765532.727873:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765532.727883:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098818816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765532.727925:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765532.727973:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765532/real 1775765532] req@ffff8b23864fea00 x1862022098818304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765532.727992:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098818304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765532.728009:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765532.728012:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765532.728016:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098818304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765532.728023:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098818432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765532.728042:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765532.728051:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765532/real 1775765532] req@ffff8b23864fd180 x1862022098818432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765532.728061:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098818432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765532.728096:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765532.728109:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765532.728112:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098818432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765532.728135:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098818560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765532.728153:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765532.728162:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765532/real 1775765532] req@ffff8b23864fce00 x1862022098818560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765532.728171:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098818560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765532.728178:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765532.728181:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765532.728183:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098818560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765532.728188:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098818688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765532.728229:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765532.728238:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765532/real 1775765532] req@ffff8b23864fe680 x1862022098818688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765532.728247:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098818688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765532.728254:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765532.728256:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765532.728259:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098818688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765532.728264:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098818816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765532.728276:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765532.728282:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765532/real 1775765532] req@ffff8b23864fca80 x1862022098818816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765532.728291:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098818816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765532.728297:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765532.728299:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765532.728301:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098818816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765533.623616:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765535.671626:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765537.719657:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765537.847602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765537.847616:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765537.847623:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765537.847638:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 2999 00000100:00080000:3.0:1775765537.847656:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765537.847665:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765537.847725:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098818944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765537.847772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765537.847777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765537.847779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765537.847784:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 2999 00000100:00080000:3.0:1775765537.847787:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765537.847790:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765537.847803:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765537.847816:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098819072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765537.847836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765537.847842:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765537.847846:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765537.847853:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2999 00000100:00080000:3.0:1775765537.847862:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765537.847876:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765537.847886:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765537.847910:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098819200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765537.847964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765537.847968:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765537.847970:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765537.847981:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2999 00000100:00080000:3.0:1775765537.847990:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765537.847997:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765537.848005:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765537.848044:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098819328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765537.848054:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765537.848058:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765537.848061:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765537.848066:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 2999 00000100:00080000:3.0:1775765537.848071:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765537.848080:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765537.848092:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765537.848111:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098819456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765539.767591:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765541.815579:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765542.967556:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765542.967569:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765542.967574:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765542.967586:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3004 00000100:00080000:3.0:1775765542.967601:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765542.967610:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765542.967673:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098819584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765542.967708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765542.967712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765542.967714:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765542.967718:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3004 00000100:00080000:3.0:1775765542.967722:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765542.967726:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765542.967737:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765542.967751:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098819712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765542.967765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765542.967771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765542.967773:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765542.967785:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3004 00000100:00080000:3.0:1775765542.967789:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765542.967792:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765542.967797:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765542.967811:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098819840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765542.967822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765542.967826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765542.967828:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765542.967832:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3004 00000100:00080000:3.0:1775765542.967836:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765542.967840:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765542.967844:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765542.967858:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098819968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765542.967869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765542.967873:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765542.967875:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765542.967880:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3004 00000100:00080000:3.0:1775765542.967883:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765542.967887:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765542.967892:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765542.967908:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098820096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765543.863595:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765545.911650:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765547.959650:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765548.087575:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765548.087589:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765548.087597:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765548.087612:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3009 00000100:00080000:3.0:1775765548.087627:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765548.087635:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765548.087684:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098820224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765548.087732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765548.087739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765548.087743:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765548.087751:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3009 00000100:00080000:3.0:1775765548.087759:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765548.087768:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765548.087784:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765548.087813:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098820352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765548.087833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765548.087839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765548.087841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765548.087847:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3009 00000100:00080000:3.0:1775765548.087853:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765548.087859:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765548.087866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765548.087888:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098820480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765548.087907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765548.087948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765548.087952:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765548.087961:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3009 00000100:00080000:3.0:1775765548.087970:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765548.087978:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765548.087991:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765548.088021:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098820608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765548.088042:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765548.088047:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765548.088049:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765548.088055:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3009 00000100:00080000:3.0:1775765548.088063:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765548.088068:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765548.088075:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765548.088094:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098820736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765550.007638:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765552.055620:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765553.207583:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765553.207599:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765553.207607:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765553.207623:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3014 00000100:00080000:3.0:1775765553.207640:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765553.207652:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765553.207718:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098820864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765553.207762:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765553.207774:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765553.207777:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765553.207785:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3014 00000100:00080000:3.0:1775765553.207794:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765553.207804:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765553.207822:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765553.207855:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098820992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765553.207902:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765553.207907:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765553.207910:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765553.207920:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3014 00000100:00080000:3.0:1775765553.207930:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765553.207938:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765553.207949:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765553.207981:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098821120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765553.208005:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765553.208012:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765553.208016:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765553.208032:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3014 00000100:00080000:3.0:1775765553.208046:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765553.208054:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765553.208064:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765553.208093:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098821248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765553.208125:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765553.208140:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765553.208143:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765553.208150:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3014 00000100:00080000:3.0:1775765553.208155:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765553.208160:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765553.208166:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765553.208184:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098821376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765554.103619:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765556.151623:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765558.199556:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765558.327576:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765558.327585:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765558.327590:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765558.327599:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3019 00000100:00080000:1.0:1775765558.327610:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765558.327615:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765558.327654:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098821504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765558.327688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765558.327692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765558.327693:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765558.327696:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3019 00000100:00080000:1.0:1775765558.327699:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765558.327701:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765558.327709:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765558.327719:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098821632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765558.327750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765558.327769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765558.327772:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765558.327785:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3019 00000100:00080000:1.0:1775765558.327791:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765558.327797:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765558.327838:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765558.327861:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098821760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765558.327904:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765558.327907:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765558.327908:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765558.327911:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3019 00000100:00080000:1.0:1775765558.327915:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765558.327921:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765558.327929:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765558.327939:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098821888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765558.327946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765558.327948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765558.327950:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765558.327952:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3019 00000100:00080000:1.0:1775765558.327955:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765558.327958:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765558.327962:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765558.328016:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098822016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765560.247618:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765562.295571:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765563.447537:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765563.447547:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765563.447551:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765563.447561:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3024 00000100:00080000:1.0:1775765563.447571:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765563.447578:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765563.447616:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098822144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765563.447644:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765563.447648:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765563.447651:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765563.447657:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3024 00000100:00080000:1.0:1775765563.447661:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765563.447667:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765563.447679:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765563.447697:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098822272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765563.447715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765563.447721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765563.447724:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765563.447730:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3024 00000100:00080000:1.0:1775765563.447737:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765563.447746:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765563.447752:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765563.447769:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098822400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765563.447797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765563.447801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765563.447803:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765563.447807:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3024 00000100:00080000:1.0:1775765563.447828:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765563.447832:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765563.447838:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765563.447858:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098822528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765563.447867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765563.447871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765563.447872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765563.447876:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3024 00000100:00080000:1.0:1775765563.447880:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765563.447883:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765563.447891:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765563.447909:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098822656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765564.343656:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765566.391641:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765568.439559:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765568.567592:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765568.567600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765568.567604:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765568.567612:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3029 00000100:00080000:1.0:1775765568.567621:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765568.567627:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765568.567664:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098822784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765568.567694:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765568.567699:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765568.567701:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765568.567705:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3029 00000100:00080000:1.0:1775765568.567709:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765568.567711:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765568.567719:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765568.567743:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098822912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765568.567756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765568.567762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765568.567763:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765568.567773:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3029 00000100:00080000:1.0:1775765568.567776:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765568.567779:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765568.567784:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765568.567795:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098823040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765568.567802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765568.567804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765568.567806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765568.567808:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3029 00000100:00080000:1.0:1775765568.567811:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765568.567813:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765568.567816:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765568.567826:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098823168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765568.567833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765568.567835:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765568.567837:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765568.567839:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3029 00000100:00080000:1.0:1775765568.567842:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765568.567844:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765568.567847:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765568.567858:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098823296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765570.487531:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765572.535647:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765573.687639:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765573.687647:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765573.687651:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765573.687659:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3035 00000100:00080000:1.0:1775765573.687669:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765573.687692:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765573.687729:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098823424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765573.687756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765573.687759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765573.687760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765573.687763:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3035 00000100:00080000:1.0:1775765573.687766:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765573.687769:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765573.687776:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765573.687785:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098823552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765573.687794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765573.687797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765573.687798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765573.687802:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3035 00000100:00080000:1.0:1775765573.687804:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765573.687806:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765573.687810:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765573.687820:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098823680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765573.687836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765573.687839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765573.687840:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765573.687844:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3035 00000100:00080000:1.0:1775765573.687847:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765573.687851:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765573.687856:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765573.687868:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098823808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765573.687880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765573.687887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765573.687889:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765573.687895:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3035 00000100:00080000:1.0:1775765573.687901:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765573.687905:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765573.687927:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765573.687947:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098823936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765574.583545:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765576.631560:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765578.679614:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765578.807588:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765578.807605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765578.807614:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765578.807684:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3040 00000100:00080000:1.0:1775765578.807705:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765578.807718:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765578.807772:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098824064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765578.807819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765578.807827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765578.807830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765578.807838:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3040 00000100:00080000:1.0:1775765578.807846:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765578.807854:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765578.807870:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765578.807898:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098824192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765578.807922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765578.807927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765578.807930:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765578.807936:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3040 00000100:00080000:1.0:1775765578.807941:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765578.807947:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765578.807955:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765578.807983:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098824320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765578.808006:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765578.808015:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765578.808019:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765578.808027:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3040 00000100:00080000:1.0:1775765578.808035:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765578.808043:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765578.808067:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765578.808095:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098824448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765578.808115:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765578.808150:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765578.808158:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765578.808172:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3040 00000100:00080000:1.0:1775765578.808183:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765578.808190:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765578.808200:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765578.808220:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098824576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765580.727589:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765582.775641:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765583.927743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765583.927757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765583.927764:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765583.927780:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3045 00000100:00080000:1.0:1775765583.927800:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765583.927811:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765583.927888:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098824704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765583.927929:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765583.927934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765583.927936:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765583.927940:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3045 00000100:00080000:1.0:1775765583.927945:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765583.927950:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765583.927963:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765583.927979:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098824832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765583.927992:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765583.927996:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765583.927999:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765583.928003:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3045 00000100:00080000:1.0:1775765583.928006:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765583.928010:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765583.928016:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765583.928036:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098824960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765583.928053:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765583.928074:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765583.928075:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765583.928079:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3045 00000100:00080000:1.0:1775765583.928082:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765583.928086:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765583.928091:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765583.928101:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098825088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765583.928111:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765583.928114:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765583.928115:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765583.928118:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3045 00000100:00080000:1.0:1775765583.928134:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765583.928138:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765583.928142:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765583.928154:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098825216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765584.823678:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765586.871602:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775765586.999731:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775765586.999770:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775765586.999787:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765586.999833:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765586.999842:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765586.999851:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765586.999861:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00010000:00010000:2.0:1775765588.919737:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765589.047795:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765589.047815:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765589.047823:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765589.047842:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3050 00000100:00080000:2.0:1775765589.047861:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765589.047871:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765589.047929:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098825344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765589.047981:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765589.047986:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765589.047989:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765589.047994:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3050 00000100:00080000:2.0:1775765589.047999:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765589.048004:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765589.048017:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765589.048032:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098825472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765589.048046:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765589.048050:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765589.048053:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765589.048058:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3050 00000100:00080000:2.0:1775765589.048063:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765589.048069:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765589.048084:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765589.048096:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098825600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765589.048109:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765589.048114:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765589.048117:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765589.048125:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3050 00000100:00080000:2.0:1775765589.048131:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765589.048144:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765589.048154:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765589.048181:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098825728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765589.048199:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765589.048205:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765589.048209:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765589.048216:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3050 00000100:00080000:2.0:1775765589.048224:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765589.048233:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765589.048244:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765589.048265:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098825856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765590.967742:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765593.015650:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765594.167645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765594.167660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765594.167667:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765594.167680:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3055 00000100:00080000:1.0:1775765594.167695:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765594.167704:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765594.167754:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098825984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765594.167796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765594.167806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765594.167809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765594.167816:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3055 00000100:00080000:1.0:1775765594.167823:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765594.167830:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765594.167844:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765594.167873:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098826112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765594.167900:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765594.167906:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765594.167909:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765594.167916:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3055 00000100:00080000:1.0:1775765594.167924:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765594.167931:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765594.167945:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765594.167978:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098826240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765594.167994:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765594.168008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765594.168014:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765594.168027:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3055 00000100:00080000:1.0:1775765594.168035:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765594.168046:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765594.168062:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765594.168087:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098826368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765594.168106:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765594.168118:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765594.168121:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765594.168130:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3055 00000100:00080000:1.0:1775765594.168137:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765594.168147:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765594.168156:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765594.168184:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098826496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765595.063550:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765597.111573:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765599.159554:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765599.287551:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765599.287561:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765599.287565:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765599.287587:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3060 00000100:00080000:3.0:1775765599.287597:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765599.287604:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765599.287642:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098826624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765599.287671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765599.287675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765599.287677:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765599.287682:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3060 00000100:00080000:3.0:1775765599.287687:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765599.287692:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765599.287702:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765599.287720:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098826752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765599.287736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765599.287740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765599.287743:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765599.287748:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3060 00000100:00080000:3.0:1775765599.287752:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765599.287757:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765599.287762:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765599.287777:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098826880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765599.287789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765599.287793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765599.287795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765599.287800:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3060 00000100:00080000:3.0:1775765599.287805:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765599.287815:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765599.287825:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765599.287842:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098827008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765599.287890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765599.287894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765599.287896:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765599.287903:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3060 00000100:00080000:3.0:1775765599.287910:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765599.287915:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765599.287922:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765599.287941:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098827136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765601.207600:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765603.255563:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765604.407573:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765604.407586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765604.407592:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765604.407606:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3065 00000100:00080000:3.0:1775765604.407640:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765604.407651:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765604.407729:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098827264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765604.407769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765604.407774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765604.407777:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765604.407782:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3065 00000100:00080000:3.0:1775765604.407786:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765604.407791:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765604.407805:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765604.407820:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098827392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765604.407853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765604.407857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765604.407859:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765604.407864:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3065 00000100:00080000:3.0:1775765604.407867:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765604.407872:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765604.407878:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765604.407896:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098827520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765604.407917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765604.407921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765604.407922:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765604.407927:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3065 00000100:00080000:3.0:1775765604.407930:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765604.407933:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765604.407939:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765604.407950:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098827648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765604.407961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765604.407965:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765604.407967:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765604.407971:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3065 00000100:00080000:3.0:1775765604.407975:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765604.407979:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765604.407984:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765604.407994:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098827776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765605.303603:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765607.351554:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765609.399604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765609.527587:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765609.527598:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765609.527603:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765609.527616:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3070 00000100:00080000:2.0:1775765609.527630:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765609.527639:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765609.527698:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022098827904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765609.527752:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765609.527760:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765609.527764:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765609.527772:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3070 00000100:00080000:2.0:1775765609.527780:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765609.527787:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765609.527813:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765609.527842:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098828032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765609.527867:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765609.527874:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765609.527878:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765609.527893:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3070 00000100:00080000:2.0:1775765609.527905:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765609.527915:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765609.527926:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765609.527965:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098828160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765609.527983:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765609.527990:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765609.527992:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765609.528000:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3070 00000100:00080000:2.0:1775765609.528007:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765609.528011:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765609.528018:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765609.528034:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098828288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765609.528046:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765609.528050:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765609.528053:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765609.528064:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3070 00000100:00080000:2.0:1775765609.528070:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765609.528081:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765609.528094:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765609.528114:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098828416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765611.447584:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765613.495605:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765614.647592:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765614.647603:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765614.647607:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765614.647618:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3076 00000100:00080000:3.0:1775765614.647631:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765614.647638:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765614.647692:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098828544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765614.647733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765614.647748:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765614.647750:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765614.647755:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3076 00000100:00080000:3.0:1775765614.647759:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765614.647763:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765614.647776:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765614.647793:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098828672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765614.647815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765614.647823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765614.647827:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765614.647833:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3076 00000100:00080000:3.0:1775765614.647837:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765614.647842:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765614.647848:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765614.647864:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098828800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765614.647875:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765614.647879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765614.647880:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765614.647884:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3076 00000100:00080000:3.0:1775765614.647888:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765614.647891:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765614.647896:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765614.647905:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098828928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765614.647918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765614.647921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765614.647922:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765614.647925:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3076 00000100:00080000:3.0:1775765614.647928:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765614.647931:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765614.647934:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765614.647941:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098829056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765615.543609:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765617.591595:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765619.639595:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765619.767548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765619.767558:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765619.767579:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765619.767590:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3081 00000100:00080000:3.0:1775765619.767602:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765619.767609:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765619.767669:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098829184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765619.767700:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765619.767703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765619.767705:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765619.767708:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3081 00000100:00080000:3.0:1775765619.767711:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765619.767714:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765619.767722:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765619.767733:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098829312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765619.767742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765619.767747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765619.767749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765619.767753:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3081 00000100:00080000:3.0:1775765619.767755:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765619.767758:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765619.767762:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765619.767769:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098829440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765619.767776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765619.767796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765619.767797:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765619.767801:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3081 00000100:00080000:3.0:1775765619.767803:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765619.767806:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765619.767810:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765619.767818:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098829568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765619.767829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765619.767831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765619.767832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765619.767835:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3081 00000100:00080000:3.0:1775765619.767838:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765619.767841:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765619.767845:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765619.767856:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098829696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765621.687657:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765623.735722:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765624.887534:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765624.887564:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765624.887568:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765624.887578:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3086 00000100:00080000:3.0:1775765624.887589:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765624.887596:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765624.887635:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098829824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765624.887674:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765624.887680:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765624.887682:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765624.887688:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3086 00000100:00080000:3.0:1775765624.887694:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765624.887699:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765624.887712:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765624.887730:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098829952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765624.887742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765624.887746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765624.887749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765624.887753:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3086 00000100:00080000:3.0:1775765624.887758:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765624.887763:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765624.887767:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765624.887782:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098830080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765624.887809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765624.887812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765624.887813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765624.887819:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3086 00000100:00080000:3.0:1775765624.887822:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765624.887826:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765624.887833:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765624.887846:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098830208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765624.887855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765624.887858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765624.887860:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765624.887875:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3086 00000100:00080000:3.0:1775765624.887879:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765624.887884:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765624.887890:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765624.887905:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098830336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765625.783667:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765627.831668:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765629.879723:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765630.007614:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765630.007628:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765630.007635:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765630.007646:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3091 00000100:00080000:1.0:1775765630.007662:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765630.007671:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765630.007720:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098830464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765630.007761:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765630.007779:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765630.007782:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765630.007787:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3091 00000100:00080000:1.0:1775765630.007792:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765630.007798:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765630.007810:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765630.007828:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098830592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765630.007843:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765630.007849:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765630.007851:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765630.007857:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3091 00000100:00080000:1.0:1775765630.007861:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765630.007869:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765630.007875:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765630.007896:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098830720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765630.007911:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765630.007916:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765630.007918:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765630.007924:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3091 00000100:00080000:1.0:1775765630.007928:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765630.007931:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765630.007938:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765630.007960:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098830848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765630.008002:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765630.008009:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765630.008014:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765630.008025:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3091 00000100:00080000:1.0:1775765630.008035:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765630.008043:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765630.008053:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765630.008086:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098830976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765631.927643:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765633.975678:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765635.127669:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765635.127716:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765635.127724:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765635.127742:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3096 00000100:00080000:2.0:1775765635.127795:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765635.127806:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765635.127883:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098831104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765635.127923:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765635.127929:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765635.127931:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765635.127935:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3096 00000100:00080000:2.0:1775765635.127940:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765635.127944:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765635.127956:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765635.127975:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098831232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765635.127995:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765635.128001:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765635.128004:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765635.128032:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3096 00000100:00080000:2.0:1775765635.128044:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765635.128052:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765635.128059:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765635.128079:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098831360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765635.128115:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765635.128120:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765635.128122:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765635.128145:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3096 00000100:00080000:2.0:1775765635.128150:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765635.128155:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765635.128162:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765635.128176:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098831488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765635.128196:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765635.128199:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765635.128201:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765635.128214:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3096 00000100:00080000:2.0:1775765635.128219:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765635.128223:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765635.128228:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765635.128241:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098831616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765636.023637:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765638.071592:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765640.119557:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765640.247559:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765640.247568:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765640.247572:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765640.247581:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3101 00000100:00080000:1.0:1775765640.247590:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765640.247596:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765640.247630:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098831744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765640.247657:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765640.247661:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765640.247663:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765640.247668:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3101 00000100:00080000:1.0:1775765640.247671:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765640.247676:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765640.247686:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765640.247701:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098831872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765640.247716:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765640.247720:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765640.247723:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765640.247728:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3101 00000100:00080000:1.0:1775765640.247733:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765640.247742:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765640.247751:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765640.247768:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098832000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765640.247781:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765640.247784:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765640.247787:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765640.247792:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3101 00000100:00080000:1.0:1775765640.247797:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765640.247805:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765640.247811:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765640.247823:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098832128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765640.247830:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765640.247833:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765640.247836:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765640.247842:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3101 00000100:00080000:1.0:1775765640.247849:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765640.247856:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765640.247860:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765640.247868:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098832256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765642.167601:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765644.215604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765645.367546:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765645.367555:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765645.367559:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765645.367567:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3106 00000100:00080000:1.0:1775765645.367576:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765645.367581:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765645.367614:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098832384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765645.367647:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765645.367651:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765645.367653:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765645.367657:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3106 00000100:00080000:1.0:1775765645.367661:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765645.367665:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765645.367675:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765645.367690:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098832512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765645.367709:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765645.367712:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765645.367713:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765645.367717:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3106 00000100:00080000:1.0:1775765645.367723:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765645.367729:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765645.367733:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765645.367748:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098832640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765645.367754:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765645.367757:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765645.367758:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765645.367760:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3106 00000100:00080000:1.0:1775765645.367762:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765645.367764:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765645.367768:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765645.367784:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098832768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765645.367790:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765645.367793:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765645.367794:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765645.367796:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3106 00000100:00080000:1.0:1775765645.367817:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765645.367820:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765645.367823:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765645.367833:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098832896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765646.263646:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765648.311573:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765650.359549:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765650.487588:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765650.487597:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765650.487601:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765650.487621:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3111 00000100:00080000:2.0:1775765650.487632:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765650.487637:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765650.487672:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022098833024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765650.487697:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765650.487700:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765650.487701:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765650.487704:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3111 00000100:00080000:2.0:1775765650.487706:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765650.487709:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765650.487716:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765650.487725:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098833152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765650.487734:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765650.487737:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765650.487739:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765650.487743:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3111 00000100:00080000:2.0:1775765650.487746:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765650.487752:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765650.487757:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765650.487765:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022098833280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765650.487772:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765650.487775:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765650.487776:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765650.487780:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3111 00000100:00080000:2.0:1775765650.487782:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765650.487785:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765650.487788:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765650.487795:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098833408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765650.487801:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765650.487804:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765650.487805:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765650.487808:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3111 00000100:00080000:2.0:1775765650.487811:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765650.487813:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765650.487816:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765650.487823:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098833536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765652.407609:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775765653.559824:0:2363:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:3.0:1775765653.559878:0:2363:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765653.559897:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775765653.559959:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775765653.559967:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775765653.559974:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775765653.559979:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00010000:00010000:2.0:1775765654.455584:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765655.607572:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765655.607589:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765655.607596:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765655.607613:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3117 00000100:00080000:2.0:1775765655.607643:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765655.607657:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765655.607716:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022098833664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765655.607785:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765655.607793:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765655.607796:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765655.607804:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3117 00000100:00080000:2.0:1775765655.607810:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765655.607817:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765655.607831:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765655.607853:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098833792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765655.607871:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765655.607880:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765655.607885:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765655.607925:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3117 00000100:00080000:2.0:1775765655.607938:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765655.607945:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765655.607956:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765655.607986:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022098833920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765655.608009:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765655.608015:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765655.608019:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765655.608031:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3117 00000100:00080000:2.0:1775765655.608047:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765655.608058:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765655.608070:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765655.608116:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098834048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765655.608158:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765655.608175:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765655.608184:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765655.608195:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3117 00000100:00080000:2.0:1775765655.608203:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765655.608217:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765655.608229:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765655.608268:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098834176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765656.503584:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765658.551621:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765660.599592:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765660.727824:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098834304:192.168.202.128@tcp:250:kworker.0 00000800:00000100:3.0:1775765660.727963:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765660.728014:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765660/real 1775765660] req@ffff8b2382dae680 x1862022098834304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765660.728032:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098834304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765660.728042:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765660.728046:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765660.728050:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098834304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765660.728058:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098834432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765660.728081:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765660.728090:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765660/real 1775765660] req@ffff8b2382dad500 x1862022098834432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765660.728100:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098834432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765660.728111:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765660.728114:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765660.728117:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098834432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765660.728140:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098834560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765660.728154:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765660.728160:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765660/real 1775765660] req@ffff8b2382daea00 x1862022098834560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765660.728169:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098834560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765660.728175:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765660.728178:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765660.728180:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098834560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765660.728185:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098834688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765660.728198:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765660.728204:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765660/real 1775765660] req@ffff8b2382daed80 x1862022098834688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765660.728213:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098834688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765660.728218:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765660.728221:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765660.728224:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098834688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765660.728228:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098834816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765660.728239:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765660.728245:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765660/real 1775765660] req@ffff8b2382dace00 x1862022098834816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765660.728253:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098834816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765660.728258:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765660.728261:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765660.728263:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098834816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765662.647616:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765664.695595:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765665.847949:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098834944:192.168.202.128@tcp:250:kworker.0 00000800:00000100:3.0:1775765665.848047:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765665.848109:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765665/real 1775765665] req@ffff8b2382dadf80 x1862022098834944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765665.848165:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098834944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765665.848216:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765665.848234:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765665.848239:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098834944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765665.848271:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098835072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765665.848303:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765665.848324:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765665/real 1775765665] req@ffff8b2382daf480 x1862022098835072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765665.848340:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098835072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765665.848348:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765665.848352:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765665.848364:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098835072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765665.848372:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098835200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765665.848390:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765665.848398:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765665/real 1775765665] req@ffff8b2382dac700 x1862022098835200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765665.848409:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098835200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765665.848416:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765665.848419:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765665.848421:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098835200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765665.848427:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098835328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765665.848441:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765665.848448:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765665/real 1775765665] req@ffff8b2382dafb80 x1862022098835328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765665.848512:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098835328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765665.848526:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765665.848531:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765665.848536:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098835328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765665.848545:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098835456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765665.848641:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765665.848660:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765665/real 1775765665] req@ffff8b2382dace00 x1862022098835456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765665.848678:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098835456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765665.848690:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765665.848695:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765665.848700:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098835456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765666.743622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765668.791659:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765670.839732:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765670.967632:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765670.967648:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765670.967654:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765670.967670:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3132 00000100:00080000:3.0:1775765670.967687:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765670.967697:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765670.967816:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098835584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765670.967863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765670.967869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765670.967871:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765670.967875:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3132 00000100:00080000:3.0:1775765670.967880:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765670.967885:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765670.967898:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765670.967911:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098835712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765670.967925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765670.967929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765670.967931:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765670.967936:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3132 00000100:00080000:3.0:1775765670.967939:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765670.967943:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765670.967947:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765670.967962:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098835840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765670.967973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765670.967976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765670.967977:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765670.967981:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3132 00000100:00080000:3.0:1775765670.967984:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765670.967987:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765670.967993:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765670.968003:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098835968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765670.968013:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765670.968017:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765670.968019:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765670.968021:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3132 00000100:00080000:3.0:1775765670.968025:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765670.968027:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765670.968031:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765670.968041:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098836096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765672.887605:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765674.935561:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765676.087899:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098836224:192.168.202.128@tcp:250:kworker.0 00000800:00000100:3.0:1775765676.087988:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765676.088045:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765676/real 1775765676] req@ffff8b2382daed80 x1862022098836224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765676.088077:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098836224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765676.088150:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765676.088164:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765676.088184:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098836224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765676.088235:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098836352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765676.088262:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765676.088272:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765676/real 1775765676] req@ffff8b2382daea00 x1862022098836352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765676.088281:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098836352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765676.088286:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765676.088289:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765676.088291:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098836352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765676.088296:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098836480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765676.088315:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765676.088322:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765676/real 1775765676] req@ffff8b2382dad500 x1862022098836480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765676.088330:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098836480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765676.088335:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765676.088337:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765676.088339:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098836480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765676.088343:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098836608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765676.088355:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765676.088360:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765676/real 1775765676] req@ffff8b2382dae680 x1862022098836608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765676.088372:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098836608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765676.088376:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765676.088378:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765676.088380:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098836608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765676.088384:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098836736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765676.088394:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765676.088399:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765676/real 1775765676] req@ffff8b2382dac380 x1862022098836736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765676.088406:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098836736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765676.088411:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765676.088413:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765676.088414:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098836736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765676.983539:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765679.031563:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765681.079640:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765681.207917:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098836864:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775765681.207919:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3142 00000100:00080000:0.0:1775765681.207927:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765681.207932:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765681.207937:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765681.207951:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098837120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765681.207960:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765681.207964:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765681.207967:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765681.207971:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3142 00000100:00080000:0.0:1775765681.207974:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765681.207977:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765681.207981:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765681.207991:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098837248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765681.207999:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765681.208003:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765681.208004:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765681.208007:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3142 00000100:00080000:0.0:1775765681.208011:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765681.208014:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765681.208018:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765681.208024:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765681.208033:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098837376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765681.208078:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765681/real 1775765681] req@ffff8b2382dad880 x1862022098836864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765681.208101:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098836864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765681.208119:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765681.208126:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765681.208131:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098836864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765681.208167:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098836992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765681.208192:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765681.208205:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765681/real 1775765681] req@ffff8b2382daca80 x1862022098836992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765681.208216:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098836992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765681.208223:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765681.208226:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765681.208229:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098836992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765681.208235:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098837120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765681.208249:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765681.208256:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765681/real 1775765681] req@ffff8b2382daf100 x1862022098837120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765681.208266:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098837120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765681.208274:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765681.208276:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765681.208278:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098837120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765681.208284:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098837248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765681.208317:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765681.208325:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765681/real 1775765681] req@ffff8b2382dac000 x1862022098837248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765681.208334:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098837248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765681.208341:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765681.208343:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765681.208345:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098837248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765681.208350:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098837376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765681.208362:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765681.208368:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765681/real 1775765681] req@ffff8b2382dac380 x1862022098837376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765681.208379:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098837376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765681.208390:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765681.208394:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765681.208398:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098837376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765682.103633:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765683.127591:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765684.151632:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765685.175700:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765686.199764:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765686.327626:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765686.327634:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765686.327638:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765686.327646:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3147 00000100:00080000:3.0:1775765686.327656:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765686.327661:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765686.327694:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098837504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765686.327729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765686.327733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765686.327735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765686.327739:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3147 00000100:00080000:3.0:1775765686.327743:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765686.327748:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765686.327758:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765686.327773:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098837632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765686.327784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765686.327787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765686.327788:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765686.327793:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3147 00000100:00080000:3.0:1775765686.327796:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765686.327805:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765686.327808:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098837504:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775765686.327811:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765686.327837:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098837760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765686.327851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765686.327854:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765686.327855:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765686.327859:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3147 00000100:00080000:3.0:1775765686.327861:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765686.327863:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765686.327866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765686.327876:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098837888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765686.327882:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765686.327886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765686.327887:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765686.327890:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3147 00000100:00080000:3.0:1775765686.327896:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765686.327898:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765686.327902:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765686.327908:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098838016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765686.327937:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765686.327997:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765686/real 1775765686] req@ffff8b23864f1f80 x1862022098837504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765686.328024:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098837504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765686.328040:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765686.328047:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765686.328052:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098837504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765686.328086:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098837632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765686.328114:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765686.328129:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765686/real 1775765686] req@ffff8b23864f0380 x1862022098837632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765686.328143:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098837632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765686.328153:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765686.328157:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765686.328161:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098837632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765686.328172:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098837760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765686.328193:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765686.328205:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765686/real 1775765686] req@ffff8b23864f1c00 x1862022098837760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765686.328219:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098837760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765686.328228:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765686.328232:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765686.328235:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098837760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765686.328242:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098837888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765686.328269:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765686.328279:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765686/real 1775765686] req@ffff8b23864f2680 x1862022098837888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765686.328293:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098837888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765686.328302:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765686.328306:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765686.328309:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098837888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765686.328316:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098838016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765686.328335:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765686.328345:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765686/real 1775765686] req@ffff8b23864f0a80 x1862022098838016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765686.328359:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098838016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765686.328369:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765686.328372:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765686.328376:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098838016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765687.223701:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765688.247610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765689.271624:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765690.295683:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765691.319550:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765691.447512:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765691.447521:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765691.447524:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765691.447532:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3152 00000100:00080000:3.0:1775765691.447543:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765691.447548:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765691.447596:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098838144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765691.447618:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765691.447621:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765691.447622:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765691.447625:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3152 00000100:00080000:3.0:1775765691.447627:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765691.447630:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765691.447637:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765691.447647:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098838272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765691.447655:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765691.447658:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765691.447659:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765691.447664:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3152 00000100:00100000:0.0:1775765691.447667:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098838144:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765691.447667:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765691.447672:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765691.447678:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765691.447686:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098838400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765691.447692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765691.447695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765691.447696:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765691.447699:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3152 00000100:00080000:3.0:1775765691.447702:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765691.447704:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765691.447707:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765691.447713:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098838528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765691.447719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765691.447721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765691.447722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765691.447725:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3152 00000100:00080000:3.0:1775765691.447728:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775765691.447731:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775765691.447734:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765691.447738:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765691.447746:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098838656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765691.447775:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765691/real 1775765691] req@ffff8b23864f1180 x1862022098838144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765691.447795:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098838144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765691.447806:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765691.447810:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765691.447814:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098838144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765691.447821:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098838272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765691.447838:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765691.447847:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765691/real 1775765691] req@ffff8b23864f0700 x1862022098838272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765691.447855:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098838272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765691.447861:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765691.447863:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765691.447865:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098838272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765691.447884:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098838400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765691.447894:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765691.447899:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765691/real 1775765691] req@ffff8b23864f0000 x1862022098838400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765691.447906:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098838400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765691.447911:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765691.447913:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765691.447915:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098838400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765691.447918:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098838528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765691.447928:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765691.447933:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765691/real 1775765691] req@ffff8b23864f3100 x1862022098838528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765691.447941:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098838528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765691.447945:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765691.447947:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765691.447949:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098838528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765691.447952:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098838656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765691.447962:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765691.447967:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765691/real 1775765691] req@ffff8b23864f2a00 x1862022098838656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765691.447973:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098838656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765691.447978:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765691.447980:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765691.447981:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098838656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765692.343648:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765693.367591:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765694.391687:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765695.415647:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765696.439683:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765696.567596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765696.567604:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765696.567607:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765696.567615:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3157 00000100:00080000:0.0:1775765696.567625:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765696.567631:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765696.567661:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098838784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765696.567691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765696.567696:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765696.567698:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765696.567702:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3157 00000100:00080000:0.0:1775765696.567707:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765696.567711:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765696.567722:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765696.567730:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098838784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765696.567737:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098838912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765696.567746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765696.567749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765696.567751:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765696.567755:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3157 00000100:00080000:0.0:1775765696.567759:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765696.567763:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765696.567768:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765696.567783:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098839040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765696.567794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765696.567807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765696.567809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775765696.567816:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775765696.567819:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3157 00000100:00080000:0.0:1775765696.567824:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765696.567827:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765696.567832:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765696.567843:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098839168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765696.567850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765696.567852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765696.567853:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765696.567857:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3157 00000100:00080000:0.0:1775765696.567860:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765696.567863:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765696.567866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765696.567870:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765696/real 1775765696] req@ffff8b2382dae680 x1862022098838784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765696.567877:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098839296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765696.567906:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098838784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765696.567920:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765696.567924:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765696.567928:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098838784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765696.567947:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098838912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765696.567966:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765696.567975:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765696/real 1775765696] req@ffff8b2382dad500 x1862022098838912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765696.567983:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098838912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765696.567988:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765696.567991:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765696.567993:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098838912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765696.567997:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098839040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765696.568019:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765696.568026:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765696/real 1775765696] req@ffff8b2382daea00 x1862022098839040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765696.568033:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098839040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765696.568039:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765696.568041:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765696.568043:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098839040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765696.568047:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098839168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765696.568059:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765696.568064:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765696/real 1775765696] req@ffff8b2382daed80 x1862022098839168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765696.568071:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098839168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765696.568076:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765696.568077:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765696.568079:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098839168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765696.568082:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098839296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765696.568092:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765696.568097:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765696/real 1775765696] req@ffff8b2382dace00 x1862022098839296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765696.568104:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098839296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765696.568109:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765696.568110:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765696.568112:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098839296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765697.463636:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765698.487639:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765699.511659:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765700.535632:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765701.559656:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765701.687568:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765701.687579:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765701.687585:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765701.687596:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3163 00000100:00080000:3.0:1775765701.687611:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765701.687619:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765701.687671:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098839424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765701.687706:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765701.687710:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765701.687713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765701.687719:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3163 00000100:00080000:3.0:1775765701.687724:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765701.687729:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765701.687740:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098839424:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775765701.687741:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765701.687761:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098839552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765701.687776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765701.687781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765701.687782:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765701.687788:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3163 00000100:00080000:3.0:1775765701.687792:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765701.687797:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765701.687803:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765701.687817:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098839680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765701.687827:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775765701.687840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765701.687845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765701.687847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765701.687854:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3163 00000100:00080000:3.0:1775765701.687859:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765701.687863:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765701.687871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765701.687875:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765701/real 1775765701] req@ffff8b23864f2d80 x1862022098839424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765701.687885:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098839808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765701.687894:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098839424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765701.687895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765701.687903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765701.687907:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765701.687911:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3163 00000100:00080000:0.0:1775765701.687913:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765701.687915:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765701.687919:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00080000:3.0:1775765701.687921:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765701.687925:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098839424:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775765701.687926:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765701.687942:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098839936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765701.687956:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098839552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765701.687976:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765701.687985:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765701/real 1775765701] req@ffff8b23864f3b80 x1862022098839552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765701.687993:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098839552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765701.687999:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765701.688001:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765701.688004:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098839552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765701.688007:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098839680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765701.688018:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765701.688023:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765701/real 1775765701] req@ffff8b23864f3480 x1862022098839680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765701.688030:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098839680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765701.688036:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765701.688037:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765701.688039:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098839680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765701.688042:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098839808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765701.688057:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765701.688062:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765701/real 1775765701] req@ffff8b23864f2a00 x1862022098839808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765701.688074:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098839808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765701.688079:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765701.688081:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765701.688083:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098839808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765701.688092:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098839936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765701.688102:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765701.688107:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765701/real 1775765701] req@ffff8b23864f3100 x1862022098839936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765701.688113:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098839936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765701.688118:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765701.688120:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765701.688122:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098839936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765702.583630:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765703.607569:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765704.631641:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765705.655622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765706.679664:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765706.807545:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765706.807553:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765706.807557:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765706.807565:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3168 00000100:00080000:3.0:1775765706.807575:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765706.807580:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765706.807615:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098840064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765706.807638:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765706.807641:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765706.807643:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765706.807645:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3168 00000100:00080000:3.0:1775765706.807648:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765706.807650:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765706.807657:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765706.807667:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098840192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765706.807674:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765706.807677:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765706.807678:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765706.807681:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3168 00000100:00080000:3.0:1775765706.807684:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765706.807686:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765706.807690:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765706.807697:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098840320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765706.807703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765706.807706:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765706.807708:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765706.807710:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3168 00000100:00080000:3.0:1775765706.807713:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765706.807716:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765706.807720:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765706.807726:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098840448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765706.807738:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098840064:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765706.807740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765706.807753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765706.807753:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765706.807756:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3168 00000100:00080000:3.0:1775765706.807758:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765706.807768:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765706.807771:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765706.807777:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098840576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765706.807845:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765706.807898:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765706/real 1775765706] req@ffff8b23864f0000 x1862022098840064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765706.807926:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098840064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765706.807936:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765706.807940:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765706.807943:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098840064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765706.807950:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098840192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765706.807969:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765706.807979:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765706/real 1775765706] req@ffff8b23864f0700 x1862022098840192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765706.807987:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098840192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765706.807992:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765706.807995:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765706.807996:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098840192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765706.808000:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098840320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765706.808015:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765706.808021:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765706/real 1775765706] req@ffff8b23864f1180 x1862022098840320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765706.808029:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098840320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765706.808039:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765706.808042:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765706.808044:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098840320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765706.808070:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098840448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765706.808081:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765706.808087:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765706/real 1775765706] req@ffff8b23864f0a80 x1862022098840448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765706.808095:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098840448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765706.808100:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765706.808102:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765706.808104:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098840448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765706.808107:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098840576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765706.808118:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765706.808124:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765706/real 1775765706] req@ffff8b23864f2680 x1862022098840576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765706.808131:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098840576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765706.808136:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765706.808138:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765706.808141:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098840576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765707.703622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765708.727674:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765709.751617:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765710.775695:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765711.799678:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765711.927547:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765711.927557:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765711.927562:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765711.927571:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3173 00000100:00080000:3.0:1775765711.927583:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765711.927589:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765711.927625:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098840704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765711.927649:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765711.927653:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765711.927655:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765711.927658:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3173 00000100:00080000:3.0:1775765711.927661:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765711.927664:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765711.927674:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765711.927684:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098840704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765711.927688:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098840832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765711.927702:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765711.927705:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765711.927706:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765711.927710:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3173 00000100:00080000:3.0:1775765711.927712:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765711.927715:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765711.927718:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765711.927728:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098840960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765711.927735:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765711.927738:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765711.927739:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765711.927742:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3173 00000100:00080000:3.0:1775765711.927746:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775765711.927751:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775765711.927752:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765711.927762:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765711.927777:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098841088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765711.927790:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765711.927792:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765711/real 1775765711] req@ffff8b23864f1c00 x1862022098840704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765711.927800:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765711.927806:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765711.927815:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3173 00000100:00100000:0.0:1775765711.927820:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098840704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765711.927821:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765711.927833:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:0.0:1775765711.927837:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1775765711.927841:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775765711.927844:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765711.927850:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098840704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765711.927859:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098841216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765711.927875:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098840832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765711.927896:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765711.927906:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765711/real 1775765711] req@ffff8b23864f0380 x1862022098840832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765711.927926:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098840832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765711.927933:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765711.927936:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765711.927939:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098840832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765711.927944:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098840960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765711.927958:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765711.927966:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765711/real 1775765711] req@ffff8b23864f1f80 x1862022098840960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765711.927975:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098840960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765711.927981:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765711.927983:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765711.927985:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098840960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765711.927990:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098841088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765711.928003:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765711.928010:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765711/real 1775765711] req@ffff8b23864f2680 x1862022098841088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765711.928019:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098841088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765711.928025:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765711.928027:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765711.928029:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098841088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765711.928034:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098841216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765711.928045:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765711.928051:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765711/real 1775765711] req@ffff8b23864f0a80 x1862022098841216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765711.928065:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098841216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765711.928071:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765711.928073:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765711.928075:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098841216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765712.823672:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765713.847631:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765714.871707:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765715.895592:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765716.919634:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765717.047571:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765717.047585:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765717.047591:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765717.047604:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3178 00000100:00080000:2.0:1775765717.047619:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765717.047640:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765717.047689:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022098841344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765717.047740:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765717.047746:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765717.047749:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765717.047757:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3178 00000100:00080000:2.0:1775765717.047763:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765717.047770:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765717.047784:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765717.047806:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098841472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765717.047810:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098841344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775765717.047839:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765717.047845:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765717.047848:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765717.047855:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3178 00000100:00080000:2.0:1775765717.047862:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765717.047881:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765717.047888:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765717.047904:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098841600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765717.047915:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765717.047919:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765717.047921:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765717.047925:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3178 00000100:00080000:2.0:1775765717.047931:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765717.047936:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765717.047941:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765717.047955:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098841472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765717.047957:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098841728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765717.047977:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765717.047980:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765717.047982:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765717.047988:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3178 00000100:00080000:2.0:1775765717.047993:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765717.047997:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765717.048003:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765717.048006:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098841600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:2.0:1775765717.048021:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098841856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765717.048042:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098841728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765717.048065:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098841856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765717.943590:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765718.967699:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765719.991553:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775765720.119710:0:2364:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:3.0:1775765720.119746:0:2364:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765720.119759:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775765720.119787:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775765720.119794:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775765720.119800:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775765720.119807:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775765720.119829:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765717/real 1775765720] req@ffff8b23b9ea5500 x1862022098841728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765720.119850:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022098841728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765720.119860:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765720.119863:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765720.119867:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098841728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765720.119873:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765717/real 1775765720] req@ffff8b23b9ea4700 x1862022098841856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765720.119880:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022098841856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765720.119885:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765720.119888:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765720.119892:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098841856:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765720.119899:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765717/real 1775765720] req@ffff8b23b9ea6a00 x1862022098841472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765720.119912:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022098841472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765720.119921:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765720.119924:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765720.119928:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098841472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765720.119935:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765717/real 1775765720] req@ffff8b23b9ea4000 x1862022098841600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765720.119948:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022098841600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765720.119964:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765720.119967:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765720.119970:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098841600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765720.119976:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765717/real 1775765720] req@ffff8b23b9ea5880 x1862022098841344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765720.119989:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022098841344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765720.119994:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765720.119997:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765720.119999:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098841344:192.168.202.128@tcp:250:kworker.0 00010000:00010000:0.0:1775765721.015701:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765722.039619:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765722.167621:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765722.167632:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765722.167637:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765722.167650:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3183 00000100:00080000:0.0:1775765722.167663:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765722.167671:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765722.167713:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098841984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765722.167753:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765722.167758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765722.167760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765722.167764:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3183 00000100:00080000:0.0:1775765722.167768:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765722.167772:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765722.167784:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765722.167799:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098842112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765722.167813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765722.167817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765722.167819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765722.167830:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3183 00000100:00100000:3.0:1775765722.167833:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098841984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775765722.167836:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765722.167849:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765722.167858:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765722.167881:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098842240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765722.167899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765722.167905:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765722.167909:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765722.167919:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3183 00000100:00080000:0.0:1775765722.167926:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775765722.167934:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775765722.167968:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765722.167978:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765722.168003:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098842368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765722.168014:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765722/real 1775765722] req@ffff8b2382dafb80 x1862022098841984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765722.168028:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765722.168043:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765722.168050:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765722.168061:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3183 00000100:00080000:0.0:1775765722.168069:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775765722.168070:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098841984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765722.168083:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765722.168101:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775765722.168107:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765722.168119:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765722.168128:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098842496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765722.168132:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098841984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765722.168161:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098842112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765722.168221:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765722.168245:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765722/real 1775765722] req@ffff8b2382dac700 x1862022098842112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765722.168297:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098842112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765722.168311:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765722.168316:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765722.168321:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098842112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765722.168362:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098842240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765722.168396:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765722.168414:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765722/real 1775765722] req@ffff8b2382daf480 x1862022098842240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765722.168433:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098842240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765722.168491:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765722.168498:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765722.168504:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098842240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765722.168514:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098842368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765722.168564:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765722.168585:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765722/real 1775765722] req@ffff8b2382dadf80 x1862022098842368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765722.168605:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098842368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765722.169087:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765722.169094:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765722.169100:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098842368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765722.169112:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098842496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765722.169155:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765722.169178:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765722/real 1775765722] req@ffff8b2382dace00 x1862022098842496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765722.169199:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098842496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765722.169215:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765722.169220:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765722.169226:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098842496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765723.063697:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765724.087541:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765725.111662:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765726.135549:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765727.159768:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765727.287614:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765727.287631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765727.287639:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765727.287655:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3188 00000100:00080000:0.0:1775765727.287678:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765727.287691:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765727.287766:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098842624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765727.287806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765727.287811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765727.287813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765727.287819:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3188 00000100:00080000:0.0:1775765727.287826:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765727.287833:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765727.287848:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765727.287863:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098842624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765727.287869:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098842752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765727.287888:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765727.287892:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765727.287894:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765727.287900:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3188 00000100:00080000:0.0:1775765727.287904:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765727.287908:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765727.287912:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765727.287930:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098842880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765727.287941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765727.287945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765727.287947:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765727.287951:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3188 00000100:00080000:0.0:1775765727.287958:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765727.287962:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765727.287967:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765727.287976:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765727.287984:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098843008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765727.287996:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765727.288001:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765727.288004:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765727.288009:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3188 00000100:00080000:0.0:1775765727.288013:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765727.288018:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765727.288023:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765727.288043:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765727/real 1775765727] req@ffff8b2382daed80 x1862022098842624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765727.288047:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098843136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765727.288099:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098842624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765727.288115:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765727.288122:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765727.288128:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098842624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765727.288165:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098842752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765727.288191:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765727.288204:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765727/real 1775765727] req@ffff8b2382daea00 x1862022098842752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765727.288252:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098842752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765727.288282:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765727.288289:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765727.288296:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098842752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765727.288308:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098842880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765727.288344:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765727.288359:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765727/real 1775765727] req@ffff8b2382dad500 x1862022098842880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765727.288373:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098842880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765727.288382:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765727.288386:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765727.288390:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098842880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765727.288396:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098843008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765727.288417:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765727.288427:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765727/real 1775765727] req@ffff8b2382dae680 x1862022098843008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765727.288458:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098843008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765727.288503:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765727.288507:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765727.288511:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098843008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765727.288519:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098843136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765727.288548:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765727.288561:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765727/real 1775765727] req@ffff8b2382dac380 x1862022098843136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765727.288576:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098843136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765727.288587:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765727.288591:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765727.288595:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098843136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765728.183655:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765729.207669:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765730.231675:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765731.255702:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765732.279578:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765732.407580:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765732.407590:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765732.407594:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765732.407604:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3193 00000100:00080000:3.0:1775765732.407617:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765732.407623:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765732.407668:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098843264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765732.407697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765732.407700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765732.407701:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765732.407704:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3193 00000100:00080000:3.0:1775765732.407706:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765732.407709:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765732.407719:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765732.407727:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098843392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765732.407738:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098843264:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765732.407738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765732.407745:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765732.407748:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765732.407752:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3193 00000100:00080000:3.0:1775765732.407754:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765732.407757:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765732.407760:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765732.407770:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098843520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765732.407776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765732.407779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765732.407781:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765732.407783:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3193 00000100:00080000:3.0:1775765732.407785:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765732.407788:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765732.407791:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765732.407797:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098843648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765732.407802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765732.407805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765732.407806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765732.407809:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3193 00000100:00080000:3.0:1775765732.407812:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765732.407814:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775765732.407819:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:3.0:1775765732.407820:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765732.407830:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098843776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765732.407874:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765732/real 1775765732] req@ffff8b23864f1180 x1862022098843264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765732.407898:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098843264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765732.407911:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765732.407916:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765732.407921:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098843264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765732.407949:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098843392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765732.407969:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765732.407979:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765732/real 1775765732] req@ffff8b23864f0700 x1862022098843392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765732.407988:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098843392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765732.407998:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765732.408001:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765732.408003:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098843392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765732.408007:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098843520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765732.408020:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765732.408025:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765732/real 1775765732] req@ffff8b23864f0000 x1862022098843520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765732.408033:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098843520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765732.408038:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765732.408040:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765732.408041:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098843520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765732.408045:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098843648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765732.408057:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765732.408063:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765732/real 1775765732] req@ffff8b23864f3100 x1862022098843648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765732.408070:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098843648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765732.408075:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765732.408077:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765732.408080:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098843648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765732.408083:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098843776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765732.408094:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765732.408099:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765732/real 1775765732] req@ffff8b23864f2a00 x1862022098843776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765732.408106:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098843776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765732.408111:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765732.408113:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765732.408115:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098843776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765733.303606:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765734.327563:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765735.351678:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765736.375590:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765737.399596:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765737.527544:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765737.527552:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765737.527556:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765737.527564:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3198 00000100:00080000:2.0:1775765737.527573:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765737.527579:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765737.527613:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098843904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765737.527636:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765737.527639:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765737.527640:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765737.527643:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3198 00000100:00080000:2.0:1775765737.527645:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765737.527648:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765737.527656:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765737.527664:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098844032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765737.527672:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098843904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775765737.527676:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765737.527681:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765737.527683:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765737.527688:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3198 00000100:00080000:2.0:1775765737.527692:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765737.527697:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765737.527702:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765737.527715:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098844160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765737.527727:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775765737.527729:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775765737.527733:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765737.527736:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765737.527741:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3198 00000100:00080000:2.0:1775765737.527745:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765737.527749:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765737.527753:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765737.527763:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098844288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765737.527769:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765737/real 1775765737] req@ffff8b23b9ea4a80 x1862022098843904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765737.527771:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765737.527778:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765737.527781:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765737.527788:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3198 00000100:00080000:2.0:1775765737.527791:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765737.527794:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765737.527797:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765737.527798:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022098843904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765737.527806:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098844416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765737.527818:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765737.527825:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765737.527830:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098843904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765737.527857:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098844032:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765737.527877:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765737.527886:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765737/real 1775765737] req@ffff8b23b9ea7100 x1862022098844032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765737.527904:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022098844032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765737.527910:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765737.527913:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765737.527915:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098844032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765737.527919:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098844160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765737.527932:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765737.527938:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765737/real 1775765737] req@ffff8b23b9ea4380 x1862022098844160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765737.527945:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022098844160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765737.527950:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765737.527952:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765737.527954:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098844160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765737.527957:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098844288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765737.527969:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765737.527975:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765737/real 1775765737] req@ffff8b23b9ea4e00 x1862022098844288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765737.527983:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022098844288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765737.527993:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765737.527995:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765737.527997:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098844288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765737.528001:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098844416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765737.528012:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765737.528018:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765737/real 1775765737] req@ffff8b23b9ea7480 x1862022098844416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765737.528025:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022098844416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765737.528030:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765737.528032:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765737.528034:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098844416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765738.423619:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765739.447721:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765740.471647:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765741.495610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765742.519619:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765742.647605:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765742.647625:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765742.647628:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765742.647637:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3204 00000100:00080000:3.0:1775765742.647647:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765742.647653:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765742.647696:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098844544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765742.647719:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765742.647722:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765742.647724:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765742.647726:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3204 00000100:00080000:3.0:1775765742.647729:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765742.647731:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765742.647737:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765742.647747:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098844672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765742.647761:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765742.647763:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765742.647765:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765742.647768:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3204 00000100:00080000:3.0:1775765742.647791:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765742.647800:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765742.647804:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098844544:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775765742.647806:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765742.647827:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098844800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765742.647836:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765742.647840:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765742.647841:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765742.647844:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3204 00000100:00080000:3.0:1775765742.647851:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765742.647853:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765742.647856:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765742.647867:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098844928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765742.647874:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765742.647877:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765742.647878:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765742.647881:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3204 00000100:00080000:3.0:1775765742.647884:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765742.647886:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765742.647890:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765742.647895:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098845056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765742.647919:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765742.647980:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765742/real 1775765742] req@ffff8b23864f3480 x1862022098844544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765742.648004:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098844544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765742.648019:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765742.648025:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765742.648030:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098844544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765742.648039:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098844672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765742.648085:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765742.648103:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765742/real 1775765742] req@ffff8b23864f3b80 x1862022098844672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765742.648117:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098844672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765742.648127:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765742.648131:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765742.648134:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098844672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765742.648166:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098844800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765742.648203:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765742.648214:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765742/real 1775765742] req@ffff8b23864f2d80 x1862022098844800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765742.648227:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098844800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765742.648236:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765742.648240:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765742.648243:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098844800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765742.648251:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098844928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765742.648270:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765742.648279:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765742/real 1775765742] req@ffff8b23864f2a00 x1862022098844928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765742.648291:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098844928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765742.648300:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765742.648303:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765742.648306:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098844928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765742.648312:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098845056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765742.648330:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765742.648339:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765742/real 1775765742] req@ffff8b23864f3100 x1862022098845056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765742.648351:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098845056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765742.648359:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765742.648362:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765742.648365:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098845056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765743.543601:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765744.567607:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765745.591683:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765746.615625:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765747.639629:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765747.767532:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765747.767543:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765747.767547:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765747.767555:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3209 00000100:00080000:3.0:1775765747.767564:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765747.767569:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765747.767600:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098845184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765747.767626:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765747.767628:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765747.767630:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765747.767632:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3209 00000100:00080000:3.0:1775765747.767634:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765747.767637:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765747.767644:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765747.767654:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098845312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765747.767663:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765747.767665:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765747.767667:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775765747.767670:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098845184:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765747.767673:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3209 00000100:00080000:3.0:1775765747.767677:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765747.767680:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765747.767683:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765747.767691:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098845440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765747.767699:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765747.767701:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765747.767703:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765747.767705:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3209 00000100:00080000:3.0:1775765747.767708:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765747.767710:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765747.767712:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765747.767719:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098845568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765747.767725:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765747.767727:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765747.767728:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765747.767730:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3209 00000100:00080000:3.0:1775765747.767732:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765747.767734:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775765747.767738:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:3.0:1775765747.767739:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765747.767747:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098845696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765747.767775:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765747/real 1775765747] req@ffff8b23864f0000 x1862022098845184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765747.767789:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098845184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765747.767797:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765747.767801:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765747.767804:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098845184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765747.767821:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098845312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765747.767836:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765747.767844:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765747/real 1775765747] req@ffff8b23864f0700 x1862022098845312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765747.767852:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098845312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765747.767857:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765747.767859:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765747.767861:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098845312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765747.767864:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098845440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765747.767874:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765747.767880:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765747/real 1775765747] req@ffff8b23864f1180 x1862022098845440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765747.767891:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098845440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765747.767896:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765747.767898:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765747.767900:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098845440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765747.767903:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098845568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765747.767915:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765747.767921:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765747/real 1775765747] req@ffff8b23864f0a80 x1862022098845568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765747.767928:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098845568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765747.767932:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765747.767934:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765747.767936:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098845568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765747.767939:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098845696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765747.767948:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765747.767954:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765747/real 1775765747] req@ffff8b23864f2680 x1862022098845696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765747.767961:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098845696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765747.767981:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765747.767983:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765747.767985:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098845696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765748.663630:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765749.687627:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765750.711620:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765751.735673:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765752.759633:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765752.887725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765752.887745:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765752.887753:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765752.887782:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3214 00000100:00080000:0.0:1775765752.887802:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765752.887811:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765752.887898:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098845824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765752.887939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765752.887944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765752.887946:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765752.887951:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3214 00000100:00080000:0.0:1775765752.887956:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765752.887963:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765752.888019:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765752.888036:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098845824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765752.888044:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098845952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765752.888067:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765752.888075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765752.888078:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765752.888085:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3214 00000100:00080000:0.0:1775765752.888094:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765752.888100:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765752.888106:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765752.888124:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098846080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765752.888135:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775765752.888159:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765752.888165:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765752.888168:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765752.888176:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3214 00000100:00080000:0.0:1775765752.888183:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765752.888190:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765752.888197:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765752/real 1775765752] req@ffff8b2382dac000 x1862022098845824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775765752.888202:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765752.888229:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098846208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765752.888238:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098845824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765752.888248:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765752.888262:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765752.888270:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765752.888270:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765752.888278:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098845824:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775765752.888279:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765752.888299:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3214 00000100:00080000:0.0:1775765752.888304:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765752.888309:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765752.888316:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765752.888328:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098845952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765752.888335:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098846336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765752.888359:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765752.888377:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765752/real 1775765752] req@ffff8b2382daf100 x1862022098845952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765752.888404:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098845952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765752.888413:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765752.888417:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765752.888421:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098845952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765752.888426:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098846080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765752.888443:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765752.888480:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765752/real 1775765752] req@ffff8b2382daca80 x1862022098846080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765752.888492:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098846080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765752.888508:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765752.888511:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765752.888514:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098846080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765752.888531:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098846208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765752.888552:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765752.888561:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765752/real 1775765752] req@ffff8b2382dad880 x1862022098846208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765752.888574:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098846208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765752.888586:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765752.888592:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765752.888596:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098846208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765752.888626:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098846336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765752.888658:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765752.888674:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765752/real 1775765752] req@ffff8b2382dac000 x1862022098846336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765752.888691:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098846336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765752.888703:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765752.888708:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765752.888713:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098846336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765753.783649:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765754.807564:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765755.831612:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765756.855552:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765757.879646:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765758.007527:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765758.007535:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765758.007539:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765758.007549:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3219 00000100:00080000:0.0:1775765758.007561:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765758.007569:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765758.007609:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098846464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765758.007633:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765758.007635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765758.007637:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765758.007640:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3219 00000100:00080000:0.0:1775765758.007643:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765758.007646:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765758.007653:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765758.007664:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098846592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765758.007669:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098846464:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775765758.007679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765758.007694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765758.007696:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765758.007701:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3219 00000100:00080000:0.0:1775765758.007706:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765758.007710:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765758.007715:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765758.007728:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098846720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765758.007740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765758.007744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765758.007746:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775765758.007751:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775765758.007756:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3219 00000100:00080000:0.0:1775765758.007760:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765758.007763:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765758.007768:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765758.007777:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098846848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765758.007784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765758.007787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765758.007789:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765758.007794:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3219 00000100:00100000:3.0:1775765758.007795:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765758/real 1775765758] req@ffff8b2382dac380 x1862022098846464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765758.007796:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765758.007802:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765758.007810:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765758.007821:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098846464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765758.007829:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098846976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765758.007829:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765758.007839:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765758.007843:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098846464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765758.007864:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098846592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765758.007881:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765758.007889:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765758/real 1775765758] req@ffff8b2382dae680 x1862022098846592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765758.007898:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098846592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765758.007903:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765758.007905:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765758.007907:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098846592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765758.007911:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098846720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765758.007943:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765758.007951:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765758/real 1775765758] req@ffff8b2382dad500 x1862022098846720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765758.007959:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098846720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765758.007965:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765758.007967:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765758.007969:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098846720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765758.007973:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098846848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765758.007984:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765758.007990:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765758/real 1775765758] req@ffff8b2382daea00 x1862022098846848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765758.007997:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098846848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765758.008002:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765758.008004:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765758.008006:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098846848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765758.008009:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098846976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765758.008018:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765758.008023:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765758/real 1775765758] req@ffff8b2382daed80 x1862022098846976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765758.008030:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098846976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765758.008044:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765758.008047:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765758.008049:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098846976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765758.903583:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765759.927655:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765760.951551:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765761.975745:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765762.999572:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765763.127613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765763.127635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765763.127639:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765763.127650:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3224 00000100:00080000:3.0:1775765763.127660:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765763.127667:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765763.127703:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098847104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765763.127733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765763.127736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765763.127738:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765763.127743:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3224 00000100:00080000:3.0:1775765763.127747:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765763.127752:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765763.127762:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765763.127770:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098847104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765763.127778:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098847232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765763.127799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765763.127803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765763.127806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765763.127810:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3224 00000100:00080000:3.0:1775765763.127814:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765763.127818:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765763.127824:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765763.127828:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765763.127841:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098847360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765763.127850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765763.127854:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765763.127856:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765763.127860:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3224 00000100:00080000:3.0:1775765763.127878:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775765763.127883:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765763/real 1775765763] req@ffff8b23864f1f80 x1862022098847104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765763.127888:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765763.127899:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765763.127908:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098847104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765763.127917:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098847488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765763.127919:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765763.127926:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765763.127933:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098847104:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765763.127942:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765763.127947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765763.127949:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765763.127955:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3224 00000100:00100000:0.0:1775765763.127959:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098847232:192.168.202.128@tcp:38:kworker.0 00000100:00080000:3.0:1775765763.127960:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765763.127969:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765763.127974:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765763.127986:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765763.127991:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098847616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765763.127999:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765763/real 1775765763] req@ffff8b23864f0380 x1862022098847232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765763.128012:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098847232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765763.128018:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765763.128021:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765763.128024:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098847232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765763.128031:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098847360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765763.128043:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765763.128049:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765763/real 1775765763] req@ffff8b23864f1c00 x1862022098847360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765763.128057:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098847360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765763.128061:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765763.128063:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765763.128065:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098847360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765763.128069:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098847488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765763.128082:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765763.128087:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765763/real 1775765763] req@ffff8b23864f2680 x1862022098847488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765763.128095:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098847488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765763.128099:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765763.128101:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765763.128102:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098847488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765763.128112:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098847616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765763.128122:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765763.128127:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765763/real 1775765763] req@ffff8b23864f0a80 x1862022098847616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765763.128134:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098847616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765763.128139:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765763.128141:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765763.128143:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098847616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765764.023683:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765765.047569:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765766.071734:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765767.095663:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765768.119638:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765768.247566:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765768.247574:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765768.247578:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765768.247586:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3229 00000100:00080000:3.0:1775765768.247607:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765768.247612:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765768.247646:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098847744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765768.247671:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765768.247674:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765768.247676:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765768.247690:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3229 00000100:00080000:3.0:1775765768.247692:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765768.247695:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765768.247702:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765768.247711:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098847872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765768.247720:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765768.247723:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765768.247725:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765768.247728:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3229 00000100:00080000:3.0:1775765768.247730:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765768.247733:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765768.247737:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765768.247757:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098848000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765768.247764:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765768.247767:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765768.247769:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765768.247772:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3229 00000100:00080000:3.0:1775765768.247774:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765768.247776:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765768.247779:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765768.247787:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098848128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765768.247793:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765768.247796:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765768.247797:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765768.247799:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3229 00000100:00080000:3.0:1775765768.247801:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765768.247804:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765768.247807:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765768.247845:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098848256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765768.247851:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098847744:192.168.202.128@tcp:250:kworker.0 00000800:00000100:0.0:1775765768.247956:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765768.248029:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765768/real 1775765768] req@ffff8b23864f1180 x1862022098847744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765768.248049:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098847744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765768.248062:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765768.248067:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765768.248072:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098847744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765768.248098:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098847872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765768.248138:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765768.248151:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765768/real 1775765768] req@ffff8b23864f0700 x1862022098847872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765768.248167:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098847872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765768.248174:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765768.248177:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765768.248179:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098847872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765768.248184:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098848000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765768.248200:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765768.248206:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765768/real 1775765768] req@ffff8b23864f0000 x1862022098848000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765768.248215:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098848000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765768.248221:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765768.248223:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765768.248225:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098848000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765768.248279:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098848128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765768.248295:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765768.248302:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765768/real 1775765768] req@ffff8b23864f3100 x1862022098848128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765768.248310:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098848128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765768.248320:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765768.248323:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765768.248325:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098848128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765768.248329:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098848256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765768.248339:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765768.248344:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765768/real 1775765768] req@ffff8b23864f2a00 x1862022098848256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765768.248351:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098848256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765768.248357:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765768.248374:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765768.248376:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098848256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765769.143678:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765770.167629:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765771.191748:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765772.215695:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765773.239634:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765773.367567:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765773.367575:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765773.367578:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765773.367587:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3234 00000100:00080000:2.0:1775765773.367596:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765773.367601:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765773.367633:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022098848384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765773.367657:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765773.367660:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765773.367661:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765773.367664:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3234 00000100:00080000:2.0:1775765773.367666:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765773.367675:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765773.367682:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765773.367691:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098848512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765773.367700:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765773.367702:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765773.367704:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765773.367708:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3234 00000100:00080000:2.0:1775765773.367712:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765773.367714:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765773.367717:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765773.367725:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098848640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765773.367737:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765773.367741:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765773.367745:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775765773.367750:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098848384:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775765773.367754:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3234 00000100:00080000:2.0:1775765773.367766:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765773.367786:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765773.367793:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765773.367812:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098848768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765773.367819:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765773.367822:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765773.367823:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765773.367827:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3234 00000100:00080000:2.0:1775765773.367832:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765773.367835:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765773.367839:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765773.367846:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098848896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765773.367863:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765773.367912:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765773/real 1775765773] req@ffff8b23b9ea6300 x1862022098848384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765773.367937:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022098848384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765773.367946:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765773.367951:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765773.367954:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098848384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765773.367961:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098848512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765773.367978:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765773.367987:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765773/real 1775765773] req@ffff8b23b9ea7480 x1862022098848512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765773.367996:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022098848512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765773.368001:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765773.368003:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765773.368005:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098848512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765773.368030:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098848640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765773.368042:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765773.368048:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765773/real 1775765773] req@ffff8b23b9ea4e00 x1862022098848640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765773.368056:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022098848640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765773.368061:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765773.368063:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765773.368065:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098848640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765773.368069:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098848768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765773.368082:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765773.368087:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765773/real 1775765773] req@ffff8b23b9ea4380 x1862022098848768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765773.368095:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022098848768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765773.368100:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765773.368102:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765773.368103:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098848768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765773.368107:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098848896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765773.368128:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765773.368133:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765773/real 1775765773] req@ffff8b23b9ea7100 x1862022098848896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765773.368140:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022098848896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765773.368144:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765773.368146:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765773.368148:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098848896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765774.263666:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765775.287596:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765776.311619:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765777.335671:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765778.359658:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765778.487551:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765778.487567:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765778.487574:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765778.487590:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3239 00000100:00080000:2.0:1775765778.487608:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765778.487619:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765778.487678:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098849024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765778.487751:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765778.487757:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765778.487760:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765778.487766:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3239 00000100:00080000:2.0:1775765778.487773:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765778.487780:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765778.487795:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765778.487808:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098849024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765778.487819:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022098849152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765778.487842:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765778.487849:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765778.487852:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765778.487859:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3239 00000100:00080000:2.0:1775765778.487865:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765778.487872:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775765778.487879:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:2.0:1775765778.487880:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765778.487901:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098849280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765778.487916:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765778.487922:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765778.487924:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765778.487935:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3239 00000100:00080000:2.0:1775765778.487940:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775765778.487946:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765778/real 1775765778] req@ffff8b23b9ea4a80 x1862022098849024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765778.487951:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765778.487975:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765778.488007:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022098849024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765778.488007:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098849408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765778.488046:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765778.488055:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765778.488085:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098849024:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775765778.488092:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765778.488099:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765778.488102:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765778.488109:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3239 00000100:00100000:0.0:1775765778.488114:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098849152:192.168.202.128@tcp:38:kworker.0 00000100:00080000:2.0:1775765778.488121:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765778.488131:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765778.488140:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765778.488141:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765778.488155:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765778/real 1775765778] req@ffff8b23b9ea5880 x1862022098849152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765778.488169:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098849536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765778.488179:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022098849152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765778.488227:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765778.488233:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765778.488238:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098849152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765778.488246:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098849280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765778.488267:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765778.488277:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765778/real 1775765778] req@ffff8b23b9ea4000 x1862022098849280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765778.488291:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022098849280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765778.488301:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765778.488304:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765778.488306:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098849280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765778.488310:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098849408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765778.488323:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765778.488329:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765778/real 1775765778] req@ffff8b23b9ea6a00 x1862022098849408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765778.488359:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022098849408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765778.488366:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765778.488369:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765778.488371:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098849408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765778.488386:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098849536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765778.488403:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765778.488430:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765778/real 1775765778] req@ffff8b23b9ea4700 x1862022098849536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765778.488442:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022098849536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765778.488450:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765778.488454:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765778.488458:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098849536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765779.383667:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765780.407613:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765781.431628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765782.455653:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765783.479734:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765783.607600:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765783.607611:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765783.607616:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765783.607626:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3245 00000100:00080000:0.0:1775765783.607640:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765783.607647:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765783.607707:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098849664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765783.607740:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765783.607744:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765783.607746:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765783.607751:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3245 00000100:00080000:0.0:1775765783.607755:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765783.607759:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765783.607772:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765783.607785:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098849792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765783.607797:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775765783.607800:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098849664:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775765783.607801:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765783.607803:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765783.607807:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3245 00000100:00080000:0.0:1775765783.607811:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765783.607816:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765783.607820:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765783.607833:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098849920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765783.607842:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765783.607846:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765783.607847:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765783.607851:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3245 00000100:00080000:0.0:1775765783.607855:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765783.607858:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765783.607869:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765783.607884:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098850048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765783.607893:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765783.607897:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765783.607899:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765783.607903:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3245 00000100:00080000:0.0:1775765783.607906:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765783.607910:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765783.607914:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765783.607923:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098850176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765783.607941:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098849792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765783.607977:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098849920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765783.608001:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098850048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765783.608017:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098850176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765784.503566:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765785.527546:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765786.551603:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775765786.679750:0:2365:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775765786.679790:0:2365:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775765786.679807:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775765786.679850:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775765786.679860:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775765786.679868:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775765786.679878:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:3.0:1775765786.679910:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765783/real 1775765786] req@ffff8b2382dadf80 x1862022098849792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765786.679937:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098849792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765786.679948:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765786.679954:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765786.679960:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098849792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765786.679970:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765783/real 1775765786] req@ffff8b2382daf480 x1862022098849920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765786.679983:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098849920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765786.679991:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765786.679995:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765786.679998:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098849920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765786.680005:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765783/real 1775765786] req@ffff8b2382dac700 x1862022098850048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765786.680025:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098850048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765786.680040:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765786.680045:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765786.680048:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098850048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765786.680055:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765783/real 1775765786] req@ffff8b2382dafb80 x1862022098850176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765786.680068:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098850176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765786.680077:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765786.680081:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765786.680085:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098850176:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765786.680093:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765783/real 1775765786] req@ffff8b2382dace00 x1862022098849664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765786.680106:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098849664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765786.680120:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765786.680124:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765786.680129:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098849664:192.168.202.128@tcp:250:kworker.0 00010000:00010000:2.0:1775765787.575559:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765788.599721:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765788.727609:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765788.727618:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765788.727636:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765788.727646:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3250 00000100:00080000:2.0:1775765788.727657:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765788.727664:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765788.727700:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098850304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765788.727724:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765788.727727:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765788.727729:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765788.727732:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3250 00000100:00080000:2.0:1775765788.727734:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765788.727737:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765788.727743:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765788.727751:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098850432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765788.727759:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765788.727762:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765788.727765:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765788.727768:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3250 00000100:00080000:2.0:1775765788.727771:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765788.727774:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765788.727777:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765788.727782:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098850560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765788.727789:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765788.727792:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765788.727794:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765788.727800:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3250 00000100:00080000:2.0:1775765788.727807:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775765788.727808:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098850304:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775765788.727812:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765788.727824:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765788.727832:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022098850688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765788.727838:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765788.727839:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765788.727840:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765788.727846:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3250 00000100:00080000:2.0:1775765788.727850:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765788.727853:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765788.727858:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765788.727874:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098850816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765788.727904:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765788.727954:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765788/real 1775765788] req@ffff8b23b9ea5500 x1862022098850304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765788.727973:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022098850304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765788.727987:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765788.727992:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765788.727997:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098850304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765788.728008:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098850432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765788.728037:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765788.728052:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765788/real 1775765788] req@ffff8b23b9ea6680 x1862022098850432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765788.728066:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022098850432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765788.728075:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765788.728079:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765788.728082:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098850432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765788.728122:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098850560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765788.728147:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765788.728160:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765788/real 1775765788] req@ffff8b23b9ea5f80 x1862022098850560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765788.728179:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022098850560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765788.728189:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765788.728192:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765788.728195:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098850560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765788.728201:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098850688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765788.728227:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765788.728238:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765788/real 1775765788] req@ffff8b23b9ea7b80 x1862022098850688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765788.728250:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022098850688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765788.728257:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765788.728262:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765788.728266:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098850688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765788.728273:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098850816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765788.728294:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765788.728312:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765788/real 1775765788] req@ffff8b23b9ea5c00 x1862022098850816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765788.728323:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022098850816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765788.728336:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765788.728340:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765788.728344:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098850816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765789.623544:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765790.647707:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765791.671662:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765792.695598:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765793.719589:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765793.847625:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765793.847634:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765793.847638:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765793.847646:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3255 00000100:00080000:1.0:1775765793.847656:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765793.847661:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765793.847705:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098850944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765793.847738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765793.847742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765793.847744:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765793.847748:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3255 00000100:00080000:1.0:1775765793.847753:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765793.847758:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765793.847770:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765793.847785:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098851072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765793.847796:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098850944:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775765793.847801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765793.847806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765793.847808:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765793.847815:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3255 00000100:00080000:1.0:1775765793.847818:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765793.847835:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765793.847842:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765793.847857:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098851200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765793.847866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765793.847869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765793.847870:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765793.847873:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3255 00000100:00080000:1.0:1775765793.847877:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775765793.847879:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775765793.847882:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765793.847888:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765793.847898:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098851328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765793.847914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765793.847916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765793.847918:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765793.847921:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3255 00000100:00080000:1.0:1775765793.847923:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765793.847926:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765793.847929:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765793.847931:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765793/real 1775765793] req@ffff8b23864ffb80 x1862022098850944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765793.847939:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098851456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765793.847966:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098850944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765793.847975:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765793.847979:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765793.847982:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098850944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765793.848005:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098851072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765793.848032:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765793.848065:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765793/real 1775765793] req@ffff8b23864fe300 x1862022098851072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765793.848095:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098851072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765793.848125:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765793.848128:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765793.848132:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098851072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765793.848139:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098851200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765793.848161:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765793.848171:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765793/real 1775765793] req@ffff8b23864ff100 x1862022098851200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765793.848179:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098851200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765793.848185:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765793.848189:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765793.848192:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098851200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765793.848198:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098851328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765793.848219:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765793.848230:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765793/real 1775765793] req@ffff8b23864fd500 x1862022098851328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765793.848242:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098851328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765793.848250:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765793.848253:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765793.848256:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098851328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765793.848264:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098851456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765793.848297:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765793.848307:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765793/real 1775765793] req@ffff8b23864fdc00 x1862022098851456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765793.848319:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098851456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765793.848328:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765793.848331:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765793.848335:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098851456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765794.743743:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765795.767590:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765796.791785:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765797.815741:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765798.839684:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765798.967546:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765798.967556:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765798.967560:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765798.967569:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3260 00000100:00080000:1.0:1775765798.967579:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765798.967586:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765798.967618:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098851584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765798.967641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765798.967644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765798.967646:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765798.967648:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3260 00000100:00080000:1.0:1775765798.967651:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765798.967653:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765798.967661:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765798.967672:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098851712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765798.967680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765798.967683:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765798.967685:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765798.967688:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3260 00000100:00080000:1.0:1775765798.967691:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775765798.967697:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098851584:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775765798.967698:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765798.967707:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765798.967721:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098851840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765798.967733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765798.967738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765798.967740:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765798.967744:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3260 00000100:00080000:1.0:1775765798.967757:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765798.967762:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775765798.967764:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:1.0:1775765798.967768:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765798.967780:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098851968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765798.967787:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765798.967790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765798.967792:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765798.967795:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3260 00000100:00080000:1.0:1775765798.967798:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765798.967801:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765798.967804:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765798.967810:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765798/real 1775765798] req@ffff8b23864fce00 x1862022098851584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765798.967813:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098852096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765798.967858:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098851584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765798.967867:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765798.967870:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765798.967873:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098851584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765798.967879:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098851712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765798.967897:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765798.967906:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765798/real 1775765798] req@ffff8b23864fe680 x1862022098851712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765798.967914:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098851712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765798.967924:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765798.967926:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765798.967928:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098851712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765798.967947:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098851840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765798.967957:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765798.967963:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765798/real 1775765798] req@ffff8b23864fca80 x1862022098851840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765798.967970:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098851840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765798.967976:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765798.967977:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765798.967979:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098851840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765798.967983:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098851968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765798.967993:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765798.967998:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765798/real 1775765798] req@ffff8b23864fea00 x1862022098851968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765798.968005:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098851968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765798.968010:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765798.968011:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765798.968014:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098851968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765798.968018:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098852096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765798.968027:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765798.968032:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765798/real 1775765798] req@ffff8b23864fd180 x1862022098852096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765798.968038:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098852096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765798.968043:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765798.968045:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765798.968047:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098852096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765799.863706:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765800.887783:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765801.911707:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765802.935703:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765803.959580:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765804.087590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765804.087602:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765804.087607:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765804.087619:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3265 00000100:00080000:1.0:1775765804.087631:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765804.087640:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765804.087682:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098852224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765804.087710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765804.087724:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765804.087726:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765804.087729:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3265 00000100:00080000:1.0:1775765804.087732:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765804.087735:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765804.087743:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765804.087753:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098852352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765804.087769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765804.087772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765804.087774:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765804.087777:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3265 00000100:00080000:1.0:1775765804.087781:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775765804.087784:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098852224:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775765804.087785:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765804.087793:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765804.087807:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098852480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765804.087815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765804.087818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765804.087819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765804.087823:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3265 00000100:00080000:1.0:1775765804.087825:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765804.087828:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765804.087832:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765804.087840:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098852608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765804.087847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765804.087850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765804.087851:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765804.087854:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3265 00000100:00080000:1.0:1775765804.087858:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775765804.087862:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775765804.087876:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765804.087887:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765804.087903:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098852736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765804.087924:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765804/real 1775765804] req@ffff8b23864fdf80 x1862022098852224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765804.087947:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098852224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765804.087961:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765804.087967:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765804.087973:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098852224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765804.087982:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098852352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765804.088011:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765804.088028:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765804/real 1775765804] req@ffff8b23864fc380 x1862022098852352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765804.088044:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098852352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765804.088055:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765804.088059:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765804.088064:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098852352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765804.088104:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098852480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765804.088131:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765804.088146:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765804/real 1775765804] req@ffff8b23864fed80 x1862022098852480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765804.088162:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098852480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765804.088172:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765804.088176:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765804.088181:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098852480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765804.088189:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098852608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765804.088214:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765804.088226:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765804/real 1775765804] req@ffff8b23864fc000 x1862022098852608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765804.088242:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098852608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765804.088252:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765804.088268:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765804.088273:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098852608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765804.088280:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098852736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765804.088305:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765804.088319:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765804/real 1775765804] req@ffff8b23864ff800 x1862022098852736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765804.088334:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098852736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765804.088344:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765804.088348:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765804.088351:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098852736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765804.983737:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765806.007636:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765807.031699:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765808.055640:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765809.079666:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765809.207618:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765809.207629:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765809.207635:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765809.207646:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3270 00000100:00080000:1.0:1775765809.207661:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765809.207669:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765809.207723:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098852864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765809.207758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765809.207762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765809.207763:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765809.207767:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3270 00000100:00080000:1.0:1775765809.207770:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765809.207773:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765809.207783:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765809.207797:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098852992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765809.207807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765809.207812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765809.207814:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765809.207817:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3270 00000100:00080000:1.0:1775765809.207820:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765809.207825:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765809.207827:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098852864:192.168.202.128@tcp:250:kworker.0 00000008:00100000:1.0:1775765809.207829:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765809.207840:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098853120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765809.207847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765809.207850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765809.207852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765809.207854:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3270 00000100:00080000:1.0:1775765809.207857:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765809.207859:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765809.207863:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765809.207872:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098853248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765809.207878:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765809.207880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765809.207882:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765809.207884:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3270 00000100:00080000:1.0:1775765809.207887:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765809.207889:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765809.207893:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765809.207904:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098853376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765809.207927:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765809.207988:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765809/real 1775765809] req@ffff8b23864ff800 x1862022098852864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765809.208005:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098852864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765809.208014:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765809.208018:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765809.208021:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098852864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765809.208028:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098852992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765809.208047:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765809.208059:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765809/real 1775765809] req@ffff8b23864fc000 x1862022098852992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765809.208067:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098852992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765809.208072:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765809.208074:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765809.208076:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098852992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765809.208100:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098853120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765809.208111:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765809.208117:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765809/real 1775765809] req@ffff8b23864fed80 x1862022098853120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765809.208124:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098853120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765809.208129:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765809.208131:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765809.208133:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098853120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765809.208137:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098853248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765809.208148:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765809.208153:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765809/real 1775765809] req@ffff8b23864fc380 x1862022098853248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765809.208161:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098853248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765809.208166:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765809.208167:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765809.208169:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098853248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765809.208172:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098853376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765809.208182:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765809.208187:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765809/real 1775765809] req@ffff8b23864fdf80 x1862022098853376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765809.208193:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098853376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765809.208198:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765809.208200:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765809.208202:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098853376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765810.103672:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765811.127574:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765812.151685:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765813.175632:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765814.199595:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765814.327597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765814.327612:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765814.327619:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765814.327633:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3275 00000100:00080000:3.0:1775765814.327647:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765814.327655:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765814.327744:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098853504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765814.327791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765814.327796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765814.327798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765814.327803:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3275 00000100:00080000:3.0:1775765814.327807:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765814.327811:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765814.327822:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765814.327834:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098853632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765814.327865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765814.327869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765814.327871:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765814.327875:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3275 00000100:00080000:3.0:1775765814.327878:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765814.327882:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765814.327887:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765814.327909:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098853760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765814.327941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765814.327943:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098853504:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765814.327945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765814.327948:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765814.327962:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3275 00000100:00080000:3.0:1775765814.327968:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765814.327973:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765814.327979:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765814.327993:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098853888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765814.328003:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765814.328006:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765814.328008:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765814.328013:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3275 00000100:00080000:3.0:1775765814.328017:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765814.328021:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765814.328028:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765814.328044:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098854016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765814.328074:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765814.328208:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765814/real 1775765814] req@ffff8b23864f2d80 x1862022098853504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765814.328237:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098853504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765814.328253:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765814.328260:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765814.328265:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098853504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765814.328277:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098853632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765814.328316:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765814.328334:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765814/real 1775765814] req@ffff8b23864f3b80 x1862022098853632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765814.328352:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098853632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765814.328364:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765814.328369:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765814.328373:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098853632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765814.328454:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098853760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765814.328569:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765814.328589:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765814/real 1775765814] req@ffff8b23864f3480 x1862022098853760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765814.328609:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098853760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765814.328623:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765814.328629:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765814.328634:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098853760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765814.328644:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098853888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765814.328694:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765814.328709:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765814/real 1775765814] req@ffff8b23864f2a00 x1862022098853888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765814.328734:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098853888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765814.328745:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765814.328749:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765814.328769:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098853888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765814.328781:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098854016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765814.328814:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765814.328831:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765814/real 1775765814] req@ffff8b23864f3100 x1862022098854016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765814.328850:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098854016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765814.328877:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765814.328882:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765814.328886:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098854016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765815.223580:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765816.247727:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765817.271600:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765818.295626:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765819.319633:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765819.447583:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765819.447597:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765819.447602:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765819.447611:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3280 00000100:00080000:0.0:1775765819.447622:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765819.447629:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765819.447667:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098854144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765819.447732:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765819.447737:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765819.447739:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765819.447743:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3280 00000100:00080000:0.0:1775765819.447747:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765819.447751:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765819.447777:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765819.447799:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098854272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765819.447837:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765819.447845:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1775765819.447845:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098854144:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775765819.447851:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765819.447859:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3280 00000100:00080000:0.0:1775765819.447865:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765819.447871:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765819.447879:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765819.447897:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098854400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765819.447910:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765819.447915:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765819.447918:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775765819.447925:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775765819.447926:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3280 00000100:00080000:0.0:1775765819.447935:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765819.447941:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765819.447950:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765819.447965:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098854528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765819.447974:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765819/real 1775765819] req@ffff8b2382dace00 x1862022098854144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765819.447978:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765819.447986:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765819.447989:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765819.447998:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3280 00000100:00080000:0.0:1775765819.448004:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765819.448020:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765819.448021:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098854144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775765819.448026:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765819.448043:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098854656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765819.448045:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765819.448060:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765819.448066:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098854144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765819.448113:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098854272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765819.448143:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765819.448158:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765819/real 1775765819] req@ffff8b2382dafb80 x1862022098854272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765819.448171:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098854272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765819.448180:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765819.448184:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765819.448187:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098854272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765819.448194:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098854400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765819.448211:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765819.448220:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765819/real 1775765819] req@ffff8b2382dac700 x1862022098854400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765819.448244:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098854400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765819.448253:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765819.448257:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765819.448260:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098854400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765819.448266:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098854528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765819.448288:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765819.448302:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765819/real 1775765819] req@ffff8b2382daf480 x1862022098854528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765819.448321:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098854528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765819.448369:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765819.448374:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765819.448379:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098854528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765819.448387:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098854656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765819.448424:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765819.448455:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765819/real 1775765819] req@ffff8b2382dadf80 x1862022098854656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765819.448510:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098854656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765819.448518:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765819.448521:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765819.448523:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098854656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765820.343553:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765821.367835:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765822.391598:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765823.415614:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765824.439575:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765824.567650:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765824.567664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765824.567670:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765824.567686:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3285 00000100:00080000:1.0:1775765824.567703:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765824.567712:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765824.567783:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098854784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765824.567819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765824.567822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765824.567824:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765824.567828:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3285 00000100:00080000:1.0:1775765824.567832:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765824.567835:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765824.567846:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765824.567861:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098854912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765824.567876:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765824.567880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765824.567883:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765824.567888:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3285 00000100:00080000:1.0:1775765824.567894:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765824.567898:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765824.567903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765824.567919:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098855040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765824.567921:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098854784:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775765824.567940:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765824.567945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765824.567947:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765824.567953:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3285 00000100:00080000:1.0:1775765824.567956:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765824.567960:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765824.567967:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765824.567978:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098855168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765824.567999:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765824.568003:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765824.568006:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765824.568009:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3285 00000100:00080000:1.0:1775765824.568013:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765824.568016:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765824.568021:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765824.568023:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775765824.568051:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098855296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765824.568122:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765824/real 1775765824] req@ffff8b23864fd180 x1862022098854784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765824.568149:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098854784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765824.568173:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765824.568179:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765824.568185:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098854784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765824.568195:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098854912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765824.568244:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765824.568258:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765824/real 1775765824] req@ffff8b23864fea00 x1862022098854912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765824.568272:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098854912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765824.568343:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765824.568350:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765824.568357:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098854912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765824.568404:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098855040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765824.568529:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765824.568544:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765824/real 1775765824] req@ffff8b23864fca80 x1862022098855040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765824.568583:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098855040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765824.568600:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765824.568606:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765824.568613:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098855040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765824.568627:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098855168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765824.568672:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765824.568694:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765824/real 1775765824] req@ffff8b23864fe680 x1862022098855168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765824.568731:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098855168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765824.568739:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765824.568742:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765824.568744:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098855168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765824.568749:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098855296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765824.568766:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765824.568773:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765824/real 1775765824] req@ffff8b23864fce00 x1862022098855296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765824.568780:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098855296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765824.568786:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765824.568789:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765824.568790:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098855296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765825.463680:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765826.487551:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765827.511614:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765828.535542:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765829.559667:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765829.687525:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765829.687534:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765829.687538:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765829.687546:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3291 00000100:00080000:1.0:1775765829.687555:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765829.687595:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765829.687631:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098855424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765829.687660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765829.687664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765829.687666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765829.687670:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3291 00000100:00080000:1.0:1775765829.687675:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765829.687680:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765829.687690:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765829.687705:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098855552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765829.687705:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098855424:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775765829.687721:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765829.687724:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765829.687725:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765829.687728:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3291 00000100:00080000:1.0:1775765829.687731:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765829.687734:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765829.687739:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765829.687748:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098855680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765829.687755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765829.687757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765829.687758:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765829.687760:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3291 00000100:00080000:1.0:1775765829.687771:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765829.687773:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765829.687776:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765829.687781:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775765829.687787:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098855808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765829.687794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765829.687797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765829.687798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765829.687801:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3291 00000100:00080000:1.0:1775765829.687803:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765829.687806:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765829.687809:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765829.687819:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098855936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765829.687830:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765829/real 1775765829] req@ffff8b23864fdc00 x1862022098855424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765829.687849:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098855424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765829.687858:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765829.687861:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765829.687864:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098855424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765829.687882:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098855552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765829.687899:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765829.687907:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765829/real 1775765829] req@ffff8b23864fd500 x1862022098855552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765829.687915:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098855552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765829.687920:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765829.687922:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765829.687924:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098855552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765829.687927:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098855680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765829.687937:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765829.687942:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765829/real 1775765829] req@ffff8b23864ff100 x1862022098855680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765829.687949:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098855680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765829.687958:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765829.687960:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765829.687962:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098855680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765829.687966:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098855808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765829.687975:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765829.687980:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765829/real 1775765829] req@ffff8b23864fe300 x1862022098855808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765829.687987:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098855808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765829.687992:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765829.687993:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765829.687995:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098855808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765829.687998:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098855936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765829.688007:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765829.688012:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765829/real 1775765829] req@ffff8b23864ffb80 x1862022098855936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765829.688018:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098855936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765829.688023:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765829.688025:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765829.688028:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098855936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765830.583539:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765831.607724:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765832.631574:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765833.655641:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765834.679581:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765834.807680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765834.807692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765834.807698:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765834.807710:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3296 00000100:00080000:3.0:1775765834.807724:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765834.807732:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765834.807779:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098856064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765834.807835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765834.807840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765834.807842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765834.807847:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3296 00000100:00080000:3.0:1775765834.807851:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765834.807855:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765834.807866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765834.807881:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098856192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765834.807896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765834.807901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765834.807903:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765834.807907:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3296 00000100:00080000:3.0:1775765834.807911:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765834.807916:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765834.807923:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765834.807929:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098856064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765834.807940:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098856320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765834.807953:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765834.807957:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765834.807960:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765834.807963:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3296 00000100:00080000:3.0:1775765834.807967:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765834.807971:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765834.807977:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765834.807989:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098856448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765834.808000:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765834.808004:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765834.808006:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765834.808009:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3296 00000800:00000100:0.0:1775765834.808016:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775765834.808016:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765834.808024:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765834.808031:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765834.808046:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098856576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765834.808079:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765834/real 1775765834] req@ffff8b23864f0000 x1862022098856064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765834.808103:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098856064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765834.808116:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765834.808121:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765834.808126:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098856064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765834.808136:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098856192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765834.808162:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765834.808175:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765834/real 1775765834] req@ffff8b23864f0700 x1862022098856192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765834.808236:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098856192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765834.808247:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765834.808251:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765834.808255:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098856192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765834.808282:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098856320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765834.808309:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765834.808321:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765834/real 1775765834] req@ffff8b23864f1180 x1862022098856320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765834.808335:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098856320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765834.808352:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765834.808356:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765834.808360:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098856320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765834.808366:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098856448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765834.808727:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765834.808778:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765834/real 1775765834] req@ffff8b23864f0a80 x1862022098856448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765834.808807:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098856448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765834.808823:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765834.808830:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765834.808836:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098856448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765834.808851:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098856576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765834.808914:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765834.808928:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765834/real 1775765834] req@ffff8b23864f2680 x1862022098856576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765834.808944:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098856576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765834.808954:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765834.808959:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765834.808965:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098856576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765835.703653:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765836.727569:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765837.751640:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765838.775585:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765839.799663:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765839.927547:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765839.927558:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765839.927562:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765839.927575:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3301 00000100:00080000:3.0:1775765839.927588:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765839.927595:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765839.927641:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098856704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765839.927669:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765839.927672:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765839.927673:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765839.927676:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3301 00000100:00080000:3.0:1775765839.927679:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765839.927682:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765839.927692:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765839.927704:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098856832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765839.927714:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765839.927716:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098856704:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765839.927717:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765839.927721:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765839.927728:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3301 00000100:00080000:3.0:1775765839.927730:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765839.927733:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765839.927736:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765839.927744:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098856960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765839.927751:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765839.927754:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765839.927755:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765839.927758:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3301 00000100:00080000:3.0:1775765839.927761:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765839.927763:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765839.927766:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765839.927774:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098857088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765839.927780:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765839.927792:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765839.927793:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765839.927796:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3301 00000100:00080000:3.0:1775765839.927799:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765839.927801:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765839.927804:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765839.927811:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098857216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765839.927815:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765839.927870:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765839/real 1775765839] req@ffff8b23864f1c00 x1862022098856704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765839.927887:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098856704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765839.927896:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765839.927899:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765839.927903:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098856704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765839.927928:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098856832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765839.927945:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765839.927954:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765839/real 1775765839] req@ffff8b23864f0380 x1862022098856832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765839.927962:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098856832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765839.927968:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765839.927970:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765839.927973:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098856832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765839.927979:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098856960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765839.927990:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765839.927996:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765839/real 1775765839] req@ffff8b23864f1f80 x1862022098856960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765839.928004:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098856960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765839.928009:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765839.928011:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765839.928012:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098856960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765839.928016:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098857088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765839.928026:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765839.928032:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765839/real 1775765839] req@ffff8b23864f2680 x1862022098857088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765839.928039:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098857088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765839.928044:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765839.928046:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765839.928047:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098857088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765839.928051:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098857216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765839.928061:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765839.928086:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765839/real 1775765839] req@ffff8b23864f0a80 x1862022098857216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765839.928094:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098857216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765839.928100:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765839.928102:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765839.928104:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098857216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765840.823529:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765841.847679:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765842.871588:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765843.895601:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765844.919593:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765845.047573:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765845.047581:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765845.047584:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765845.047592:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3306 00000100:00080000:3.0:1775765845.047602:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765845.047618:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765845.047651:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098857344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765845.047674:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765845.047676:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765845.047678:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765845.047681:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3306 00000100:00080000:3.0:1775765845.047683:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765845.047685:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765845.047692:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765845.047702:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098857472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765845.047712:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765845.047714:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765845.047715:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765845.047720:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3306 00000100:00100000:0.0:1775765845.047723:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098857344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765845.047723:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765845.047726:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765845.047739:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765845.047746:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098857600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765845.047756:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765845.047759:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765845.047760:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765845.047763:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3306 00000100:00080000:3.0:1775765845.047765:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765845.047768:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765845.047773:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765845.047785:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098857728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765845.047822:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765845.047828:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765845.047833:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765845.047840:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3306 00000100:00080000:3.0:1775765845.047850:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775765845.047855:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775765845.047864:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765845.047875:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765845.047901:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098857856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765845.047939:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765845/real 1775765845] req@ffff8b23864f1180 x1862022098857344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765845.047976:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098857344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765845.047997:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765845.048004:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765845.048012:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098857344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765845.048053:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098857472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765845.048138:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765845.048152:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765845/real 1775765845] req@ffff8b23864f0700 x1862022098857472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765845.048165:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098857472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765845.048173:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765845.048176:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765845.048179:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098857472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765845.048186:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098857600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765845.048207:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765845.048218:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765845/real 1775765845] req@ffff8b23864f0000 x1862022098857600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765845.048230:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098857600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765845.048239:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765845.048242:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765845.048245:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098857600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765845.048252:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098857728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765845.048282:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765845.048293:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765845/real 1775765845] req@ffff8b23864f3100 x1862022098857728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765845.048306:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098857728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765845.048315:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765845.048318:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765845.048322:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098857728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765845.048329:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098857856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765845.048359:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765845.048392:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765845/real 1775765845] req@ffff8b23864f2a00 x1862022098857856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765845.048415:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098857856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765845.048432:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765845.048439:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765845.048446:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098857856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765845.943589:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765846.967601:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765847.991576:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765849.015634:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765850.039754:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765850.167588:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765850.167600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765850.167607:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765850.167623:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3311 00000100:00080000:3.0:1775765850.167641:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765850.167651:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765850.167725:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098857984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765850.167768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765850.167773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765850.167776:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765850.167781:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3311 00000100:00080000:3.0:1775765850.167785:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765850.167789:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765850.167803:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765850.167823:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098858112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765850.167856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765850.167860:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098857984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765850.167867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765850.167877:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765850.167887:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3311 00000100:00080000:3.0:1775765850.167895:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765850.167903:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765850.167915:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765850.167951:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098858240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765850.167972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765850.167980:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765850.167983:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765850.167989:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3311 00000100:00080000:3.0:1775765850.168006:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765850.168012:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765850.168022:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765850.168081:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098858368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765850.168104:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098858112:192.168.202.128@tcp:38:kworker.0 00000100:00080000:3.0:1775765850.168114:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765850.168123:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765850.168127:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765850.168138:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3311 00000100:00080000:3.0:1775765850.168154:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765850.168164:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765850.168176:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765850.168190:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098858240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765850.168211:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098858496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765850.168240:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098858368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765850.168309:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098858496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765851.063612:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765852.087724:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765853.111573:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775765853.239682:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775765853.239725:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775765853.239742:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765853.239788:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765853.239798:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765853.239806:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765853.239816:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775765853.239852:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765850/real 1775765853] req@ffff8b23864f3100 x1862022098858496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765853.239887:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098858496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765853.239899:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765853.239904:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765853.239909:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098858496:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765853.239917:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765850/real 1775765853] req@ffff8b23864f2a00 x1862022098858368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765853.239930:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098858368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765853.239938:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765853.239942:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765853.239945:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098858368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765853.239952:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765850/real 1775765853] req@ffff8b23864f3b80 x1862022098858112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765853.239965:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098858112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765853.239973:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765853.239976:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765853.239980:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098858112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765853.239987:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765850/real 1775765853] req@ffff8b23864f2d80 x1862022098858240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765853.239998:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098858240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765853.240019:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765853.240024:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765853.240027:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098858240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765853.240033:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765850/real 1775765853] req@ffff8b23864f3480 x1862022098857984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765853.240045:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098857984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765853.240052:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765853.240055:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765853.240059:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098857984:192.168.202.128@tcp:250:kworker.0 00010000:00010000:0.0:1775765854.135691:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765855.159627:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765855.287568:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765855.287576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765855.287579:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765855.287588:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3316 00000100:00080000:0.0:1775765855.287597:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765855.287603:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765855.287633:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098858624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765855.287660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765855.287664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765855.287665:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765855.287669:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3316 00000100:00080000:0.0:1775765855.287673:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765855.287676:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765855.287685:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765855.287698:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098858752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765855.287711:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098858624:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775765855.287712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765855.287719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765855.287723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765855.287729:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3316 00000100:00080000:0.0:1775765855.287733:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765855.287738:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765855.287745:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765855.287771:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098858880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765855.287781:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765855.287785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765855.287787:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775765855.287794:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775765855.287798:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3316 00000100:00080000:0.0:1775765855.287803:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765855.287808:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765855.287815:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765855.287829:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098859008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765855.287841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775765855.287844:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765855/real 1775765855] req@ffff8b2382daed80 x1862022098858624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765855.287846:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765855.287850:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765855.287859:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3316 00000100:00080000:0.0:1775765855.287867:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765855.287871:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765855.287876:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765855.287881:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098858624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765855.287893:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098859136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765855.287901:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765855.287910:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765855.287916:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098858624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765855.287952:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098858752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765855.287980:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765855.288013:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765855/real 1775765855] req@ffff8b2382daea00 x1862022098858752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765855.288027:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098858752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765855.288036:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765855.288040:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765855.288043:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098858752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765855.288067:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098858880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765855.288095:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765855.288109:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765855/real 1775765855] req@ffff8b2382dad500 x1862022098858880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765855.288136:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098858880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765855.288144:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765855.288148:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765855.288151:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098858880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765855.288161:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098859008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765855.288183:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765855.288192:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765855/real 1775765855] req@ffff8b2382dae680 x1862022098859008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765855.288204:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098859008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765855.288212:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765855.288215:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765855.288218:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098859008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765855.288224:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098859136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765855.288245:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765855.288254:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765855/real 1775765855] req@ffff8b2382dac380 x1862022098859136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765855.288264:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098859136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765855.288271:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765855.288369:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765855.288373:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098859136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765856.183589:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765857.207579:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765858.231657:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765859.255609:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765860.279578:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765860.407567:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765860.407579:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765860.407584:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765860.407594:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3321 00000100:00080000:0.0:1775765860.407607:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765860.407615:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765860.407657:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098859264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765860.407683:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765860.407687:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765860.407689:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765860.407692:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3321 00000100:00080000:0.0:1775765860.407695:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765860.407698:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765860.407707:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765860.407718:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098859392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765860.407729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775765860.407732:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098859264:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775765860.407733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765860.407738:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765860.407745:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3321 00000100:00080000:0.0:1775765860.407748:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765860.407750:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765860.407754:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765860.407766:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098859520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765860.407774:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765860.407777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765860.407779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765860.407782:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3321 00000100:00080000:0.0:1775765860.407786:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765860.407788:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765860.407793:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765860.407802:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765860.407804:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098859648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765860.407814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765860.407817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765860.407819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765860.407822:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3321 00000100:00080000:0.0:1775765860.407825:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765860.407827:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765860.407831:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765860.407848:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098859776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765860.407848:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765860/real 1775765860] req@ffff8b2382dac000 x1862022098859264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765860.407875:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098859264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765860.407890:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765860.407896:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765860.407902:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098859264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765860.407932:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098859392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765860.407977:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765860.407993:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765860/real 1775765860] req@ffff8b2382dad880 x1862022098859392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765860.408003:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098859392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765860.408010:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765860.408013:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765860.408015:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098859392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765860.408020:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098859520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765860.408037:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765860.408044:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765860/real 1775765860] req@ffff8b2382daca80 x1862022098859520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765860.408052:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098859520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765860.408061:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765860.408065:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765860.408069:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098859520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765860.408077:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098859648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765860.408101:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765860.408114:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765860/real 1775765860] req@ffff8b2382daf100 x1862022098859648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765860.408129:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098859648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765860.408137:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765860.408140:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765860.408142:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098859648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765860.408147:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098859776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765860.408165:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765860.408173:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765860/real 1775765860] req@ffff8b2382dac380 x1862022098859776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765860.408181:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098859776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765860.408187:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765860.408189:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765860.408191:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098859776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765861.303548:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765862.327668:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765863.351623:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765864.375622:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765865.399567:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765865.527586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765865.527595:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765865.527599:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765865.527612:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3326 00000100:00080000:3.0:1775765865.527622:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765865.527629:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765865.527664:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098859904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765865.527691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765865.527694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765865.527696:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765865.527698:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3326 00000100:00080000:3.0:1775765865.527701:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765865.527704:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765865.527711:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765865.527720:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098860032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765865.527731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765865.527736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765865.527738:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775765865.527739:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098859904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765865.527745:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3326 00000100:00080000:3.0:1775765865.527755:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765865.527759:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765865.527763:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765865.527776:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098860160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765865.527786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765865.527789:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765865.527790:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765865.527793:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3326 00000100:00080000:3.0:1775765865.527795:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765865.527797:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765865.527801:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765865.527807:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098860288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765865.527812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765865.527815:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765865.527817:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765865.527820:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3326 00000100:00080000:3.0:1775765865.527823:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765865.527825:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765865.527828:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765865.527835:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765865.527841:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098860416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765865.527887:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765865/real 1775765865] req@ffff8b23864f0000 x1862022098859904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765865.527925:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098859904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765865.527937:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765865.527943:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765865.527947:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098859904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765865.527989:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098860032:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765865.528015:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765865.528026:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765865/real 1775765865] req@ffff8b23864f0700 x1862022098860032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765865.528035:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098860032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765865.528041:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765865.528044:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765865.528046:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098860032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765865.528051:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098860160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765865.528066:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765865.528072:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765865/real 1775765865] req@ffff8b23864f1180 x1862022098860160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765865.528080:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098860160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765865.528085:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765865.528087:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765865.528089:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098860160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765865.528102:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098860288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765865.528125:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765865.528132:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765865/real 1775765865] req@ffff8b23864f0a80 x1862022098860288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765865.528141:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098860288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765865.528147:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765865.528149:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765865.528151:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098860288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765865.528155:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098860416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765865.528166:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765865.528171:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765865/real 1775765865] req@ffff8b23864f2680 x1862022098860416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765865.528178:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098860416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765865.528184:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765865.528186:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765865.528188:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098860416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765866.423716:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765867.447551:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765868.471663:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765869.495590:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765870.519651:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765870.647530:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765870.647538:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765870.647542:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765870.647550:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3332 00000100:00080000:3.0:1775765870.647561:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765870.647567:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765870.647601:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098860544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765870.647632:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765870.647636:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765870.647638:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765870.647642:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3332 00000100:00080000:3.0:1775765870.647645:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765870.647649:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765870.647659:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765870.647675:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098860672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765870.647699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765870.647703:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098860544:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765870.647704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765870.647709:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765870.647715:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3332 00000100:00080000:3.0:1775765870.647719:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765870.647722:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765870.647727:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765870.647738:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098860800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765870.647745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765870.647748:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765870.647749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765870.647752:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3332 00000100:00080000:3.0:1775765870.647755:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765870.647757:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765870.647760:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765870.647765:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765870.647770:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098860928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765870.647777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765870.647780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765870.647782:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765870.647784:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3332 00000100:00080000:3.0:1775765870.647786:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765870.647789:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765870.647792:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765870.647800:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098861056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765870.647805:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765870/real 1775765870] req@ffff8b23864f1f80 x1862022098860544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765870.647838:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098860544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765870.647862:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765870.647868:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765870.647872:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098860544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765870.647896:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098860672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765870.647922:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765870.647935:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765870/real 1775765870] req@ffff8b23864f0380 x1862022098860672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765870.647943:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098860672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765870.647949:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765870.647951:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765870.647953:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098860672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765870.647957:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098860800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765870.647971:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765870.647978:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765870/real 1775765870] req@ffff8b23864f1c00 x1862022098860800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765870.647985:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098860800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765870.647995:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765870.647997:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765870.647999:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098860800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765870.648002:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098860928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765870.648014:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765870.648019:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765870/real 1775765870] req@ffff8b23864f2680 x1862022098860928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765870.648026:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098860928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765870.648031:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765870.648032:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765870.648034:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098860928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765870.648037:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098861056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765870.648046:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765870.648051:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765870/real 1775765870] req@ffff8b23864f0a80 x1862022098861056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765870.648058:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098861056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765870.648063:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765870.648065:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765870.648068:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098861056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765871.543594:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765872.567606:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765873.591560:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765874.615673:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765875.639581:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765875.767522:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765875.767530:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765875.767534:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765875.767542:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3337 00000100:00080000:3.0:1775765875.767551:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765875.767557:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765875.767596:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098861184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765875.767620:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765875.767623:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765875.767624:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765875.767630:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3337 00000100:00080000:3.0:1775765875.767633:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765875.767635:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765875.767643:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765875.767652:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098861312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765875.767662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765875.767666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775765875.767670:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098861184:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765875.767670:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765875.767676:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3337 00000100:00080000:3.0:1775765875.767682:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765875.767686:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765875.767692:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765875.767705:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098861440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765875.767717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765875.767720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765875.767722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765875.767727:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3337 00000100:00080000:3.0:1775765875.767731:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765875.767736:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775765875.767737:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:3.0:1775765875.767742:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765875.767753:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098861568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765875.767761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765875.767763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765875.767764:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765875.767767:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3337 00000100:00080000:3.0:1775765875.767771:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765875.767774:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765875.767778:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765875/real 1775765875] req@ffff8b23864f1180 x1862022098861184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775765875.767779:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765875.767805:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098861696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765875.767815:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098861184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765875.767824:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765875.767828:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765875.767832:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098861184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765875.767850:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098861312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765875.767880:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765875.767889:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765875/real 1775765875] req@ffff8b23864f0700 x1862022098861312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765875.767897:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098861312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765875.767903:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765875.767905:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765875.767907:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098861312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765875.767911:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098861440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765875.767928:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765875.767934:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765875/real 1775765875] req@ffff8b23864f0000 x1862022098861440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765875.767941:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098861440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765875.767946:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765875.767948:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765875.767950:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098861440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765875.767953:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098861568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765875.767965:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765875.767970:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765875/real 1775765875] req@ffff8b23864f3480 x1862022098861568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765875.767977:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098861568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765875.767982:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765875.767984:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765875.767985:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098861568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765875.767989:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098861696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765875.767998:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765875.768003:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765875/real 1775765875] req@ffff8b23864f2d80 x1862022098861696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765875.768015:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098861696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765875.768020:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765875.768021:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765875.768023:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098861696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765876.663630:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765877.687550:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765878.711715:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765879.735530:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765880.759631:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765880.887554:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765880.887561:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765880.887565:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765880.887574:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3342 00000100:00080000:3.0:1775765880.887584:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765880.887590:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765880.887623:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098861824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765880.887646:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765880.887651:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765880.887652:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765880.887655:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3342 00000100:00080000:3.0:1775765880.887657:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765880.887660:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765880.887668:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765880.887678:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098861824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765880.887679:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098861952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765880.887693:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765880.887695:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765880.887696:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765880.887699:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3342 00000100:00080000:3.0:1775765880.887701:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765880.887704:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765880.887707:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765880.887714:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098862080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765880.887721:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765880.887723:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765880.887725:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765880.887727:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3342 00000100:00080000:3.0:1775765880.887730:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765880.887732:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765880.887736:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765880.887756:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765880.887761:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098862208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765880.887769:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765880.887772:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765880.887773:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765880.887776:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3342 00000100:00080000:3.0:1775765880.887779:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765880.887782:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765880.887786:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765880.887795:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098862336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765880.887803:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765880/real 1775765880] req@ffff8b23864f3b80 x1862022098861824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765880.887823:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098861824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765880.887831:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765880.887834:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765880.887837:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098861824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765880.887855:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098861952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765880.887873:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765880.887881:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765880/real 1775765880] req@ffff8b23864f2a00 x1862022098861952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765880.887894:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098861952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765880.887899:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765880.887901:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765880.887903:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098861952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765880.887907:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098862080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765880.887918:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765880.887923:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765880/real 1775765880] req@ffff8b23864f3100 x1862022098862080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765880.887930:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098862080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765880.887936:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765880.887937:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765880.887939:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098862080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765880.887942:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098862208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765880.887961:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765880.887971:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765880/real 1775765880] req@ffff8b23864f2d80 x1862022098862208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765880.887983:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098862208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765880.887998:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765880.888002:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765880.888005:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098862208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765880.888012:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098862336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765880.888028:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765880.888035:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765880/real 1775765880] req@ffff8b23864f3480 x1862022098862336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765880.888042:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098862336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765880.888048:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765880.888050:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765880.888052:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098862336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765881.783588:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765882.807610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765883.831602:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765884.855684:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765885.879627:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765886.007603:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765886.007614:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765886.007619:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765886.007631:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3347 00000100:00080000:0.0:1775765886.007645:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765886.007652:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765886.007717:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098862464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765886.007768:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765886.007775:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765886.007777:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765886.007781:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3347 00000100:00080000:0.0:1775765886.007785:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765886.007788:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765886.007794:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098862464:192.168.202.128@tcp:250:kworker.0 00000008:00100000:0.0:1775765886.007800:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765886.007815:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098862592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765886.007825:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765886.007828:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765886.007830:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765886.007833:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3347 00000100:00080000:0.0:1775765886.007836:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765886.007839:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765886.007842:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765886.007852:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098862720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765886.007857:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765886.007860:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765886.007861:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765886.007865:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3347 00000100:00080000:0.0:1775765886.007868:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765886.007874:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775765886.007877:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:0.0:1775765886.007879:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765886.007888:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098862848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765886.007894:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765886.007896:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765886.007898:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765886.007901:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3347 00000100:00080000:0.0:1775765886.007904:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765886.007906:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765886.007909:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765886.007920:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098862976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765886.007940:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765886/real 1775765886] req@ffff8b2382dae680 x1862022098862464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765886.007960:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098862464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765886.007969:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765886.007973:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765886.007976:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098862464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765886.008000:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098862592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765886.008020:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765886.008030:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765886/real 1775765886] req@ffff8b2382dad500 x1862022098862592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765886.008038:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098862592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765886.008044:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765886.008046:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765886.008048:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098862592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765886.008052:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098862720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765886.008068:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765886.008078:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765886/real 1775765886] req@ffff8b2382daea00 x1862022098862720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765886.008091:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098862720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765886.008100:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765886.008104:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765886.008107:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098862720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765886.008114:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098862848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765886.008138:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765886.008150:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765886/real 1775765886] req@ffff8b2382daed80 x1862022098862848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765886.008171:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098862848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765886.008178:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765886.008180:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765886.008182:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098862848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765886.008187:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098862976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765886.008204:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765886.008210:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765886/real 1775765886] req@ffff8b2382dadf80 x1862022098862976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765886.008218:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098862976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765886.008224:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765886.008226:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765886.008228:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098862976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765886.903602:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765887.927600:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765888.951704:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765889.975587:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765890.999619:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765891.127616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765891.127627:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765891.127633:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765891.127663:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3352 00000100:00080000:3.0:1775765891.127679:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765891.127686:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765891.127728:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098863104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765891.127759:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765891.127764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765891.127766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765891.127771:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3352 00000100:00080000:3.0:1775765891.127775:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765891.127780:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765891.127792:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765891.127807:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098863232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765891.127813:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098863104:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765891.127823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765891.127827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765891.127828:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765891.127833:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3352 00000100:00080000:3.0:1775765891.127836:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765891.127840:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765891.127845:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765891.127855:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098863360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765891.127863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765891.127866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765891.127867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765891.127870:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3352 00000100:00080000:3.0:1775765891.127873:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765891.127876:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765891.127879:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765891.127887:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098863488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765891.127893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765891.127896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765891.127897:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765891.127903:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3352 00000800:00000100:0.0:1775765891.127904:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775765891.127906:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765891.127910:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765891.127913:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765891.127922:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098863616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765891.127951:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765891/real 1775765891] req@ffff8b23864f0000 x1862022098863104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765891.127968:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098863104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765891.127982:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765891.127985:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765891.127989:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098863104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765891.128010:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098863232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765891.128024:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765891.128032:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765891/real 1775765891] req@ffff8b23864f0700 x1862022098863232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765891.128040:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098863232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765891.128046:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765891.128048:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765891.128050:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098863232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765891.128053:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098863360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765891.128065:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765891.128070:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765891/real 1775765891] req@ffff8b23864f1180 x1862022098863360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765891.128078:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098863360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765891.128084:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765891.128086:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765891.128088:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098863360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765891.128094:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098863488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765891.128115:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765891.128126:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765891/real 1775765891] req@ffff8b23864f0a80 x1862022098863488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765891.128139:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098863488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765891.128148:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765891.128151:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765891.128154:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098863488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765891.128161:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098863616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765891.128180:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765891.128190:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765891/real 1775765891] req@ffff8b23864f2680 x1862022098863616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765891.128198:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098863616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765891.128203:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765891.128206:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765891.128208:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098863616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765892.023581:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765893.047597:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765894.071576:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765895.095579:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765896.119578:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765896.247569:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765896.247582:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765896.247587:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765896.247628:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3357 00000100:00080000:0.0:1775765896.247639:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765896.247646:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765896.247684:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098863744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765896.247714:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765896.247719:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765896.247721:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765896.247726:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3357 00000100:00080000:0.0:1775765896.247730:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765896.247735:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765896.247745:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765896.247757:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098863744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765896.247760:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098863872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765896.247776:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765896.247780:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765896.247782:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765896.247787:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3357 00000100:00080000:0.0:1775765896.247792:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765896.247796:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765896.247801:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765896.247816:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098864000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765896.247817:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775765896.247830:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765896.247834:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765896.247836:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765896.247841:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3357 00000100:00080000:0.0:1775765896.247845:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765896.247849:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765896.247854:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765896.247856:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765896/real 1775765896] req@ffff8b2382daf480 x1862022098863744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765896.247870:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098864128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765896.247873:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098863744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765896.247901:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765896.247902:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765896.247907:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00080000:0.0:1775765896.247909:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1775765896.247913:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098863744:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775765896.247914:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765896.247931:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3357 00000100:00080000:0.0:1775765896.247935:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775765896.247940:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098863872:192.168.202.128@tcp:38:kworker.0 00000100:00080000:0.0:1775765896.247941:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765896.247950:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765896.247964:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765896.247968:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098864256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765896.247975:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765896/real 1775765896] req@ffff8b2382dac700 x1862022098863872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765896.247992:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098863872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765896.247998:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765896.248000:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765896.248003:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098863872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765896.248009:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098864000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765896.248021:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765896.248026:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765896/real 1775765896] req@ffff8b2382dafb80 x1862022098864000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765896.248033:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098864000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765896.248038:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765896.248040:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765896.248042:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098864000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765896.248045:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098864128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765896.248055:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765896.248061:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765896/real 1775765896] req@ffff8b2382dace00 x1862022098864128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765896.248068:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098864128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765896.248073:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765896.248075:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765896.248077:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098864128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765896.248088:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098864256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765896.248098:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765896.248103:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765896/real 1775765896] req@ffff8b2382daf480 x1862022098864256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765896.248110:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098864256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765896.248115:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765896.248116:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765896.248118:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098864256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765897.143566:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765898.167545:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765899.191802:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765900.215639:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765901.239755:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765901.367679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765901.367690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765901.367695:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765901.367707:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3362 00000100:00080000:3.0:1775765901.367720:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765901.367729:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765901.367778:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098864384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765901.367829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765901.367834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765901.367837:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765901.367842:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3362 00000100:00080000:3.0:1775765901.367848:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765901.367854:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765901.367864:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765901.367876:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098864512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765901.367886:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098864384:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765901.367892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765901.367907:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765901.367909:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765901.367914:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3362 00000100:00080000:3.0:1775765901.367917:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765901.367920:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765901.367924:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765901.367935:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098864640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765901.367943:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765901.367946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765901.367947:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765901.367950:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3362 00000100:00080000:3.0:1775765901.367957:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765901.367960:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765901.367964:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765901.367972:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098864768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765901.367979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765901.367982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765901.367984:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775765901.367992:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775765901.367998:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3362 00000100:00080000:3.0:1775765901.368006:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765901.368011:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765901.368018:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765901.368041:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098864896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765901.368049:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765901/real 1775765901] req@ffff8b23864f1c00 x1862022098864384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765901.368108:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098864384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765901.368125:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765901.368133:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765901.368140:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098864384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765901.368176:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098864512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765901.368204:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765901.368216:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765901/real 1775765901] req@ffff8b23864f0380 x1862022098864512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765901.368230:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098864512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765901.368241:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765901.368245:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765901.368249:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098864512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765901.368258:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098864640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765901.368304:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765901.368324:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765901/real 1775765901] req@ffff8b23864f1f80 x1862022098864640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765901.368348:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098864640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765901.368358:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765901.368362:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765901.368365:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098864640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765901.368372:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098864768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765901.368415:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765901.368428:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765901/real 1775765901] req@ffff8b23864f2680 x1862022098864768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765901.368442:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098864768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765901.368451:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765901.368454:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765901.368457:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098864768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765901.368480:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098864896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765901.368495:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765901.368529:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765901/real 1775765901] req@ffff8b23864f0a80 x1862022098864896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765901.368539:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098864896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765901.368570:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765901.368574:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765901.368577:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098864896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765902.263611:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765903.287665:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765904.311631:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765905.335716:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765906.359693:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765906.487641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765906.487652:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765906.487656:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765906.487668:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3367 00000100:00080000:0.0:1775765906.487681:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765906.487688:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765906.487731:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098865024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765906.487770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765906.487774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765906.487777:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765906.487782:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3367 00000100:00080000:0.0:1775765906.487786:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765906.487792:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765906.487802:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765906.487819:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098865024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765906.487823:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098865152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765906.487845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765906.487850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765906.487852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765906.487860:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3367 00000100:00080000:0.0:1775765906.487867:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765906.487873:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765906.487879:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765906.487894:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098865280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765906.487907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1775765906.487911:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775765906.487917:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765906.487920:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765906.487929:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3367 00000100:00080000:0.0:1775765906.487938:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765906.487944:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765906.487952:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765906.487966:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765906/real 1775765906] req@ffff8b2382dadf80 x1862022098865024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765906.487972:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098865408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765906.488037:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765906.488041:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765906.488043:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775765906.488049:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098865024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765906.488050:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3367 00000100:00080000:0.0:1775765906.488062:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765906.488068:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765906.488073:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:3.0:1775765906.488076:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765906.488088:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098865024:192.168.202.128@tcp:250:kworker.0 00000008:00100000:0.0:1775765906.488090:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765906.488150:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098865536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765906.488160:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098865152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765906.488196:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765906.488208:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765906/real 1775765906] req@ffff8b2382daed80 x1862022098865152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765906.488218:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098865152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765906.488224:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765906.488228:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765906.488232:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098865152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765906.488237:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098865280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765906.488251:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765906.488257:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765906/real 1775765906] req@ffff8b2382daea00 x1862022098865280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765906.488264:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098865280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765906.488270:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765906.488272:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765906.488273:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098865280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765906.488277:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098865408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765906.488290:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765906.488295:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765906/real 1775765906] req@ffff8b2382dad500 x1862022098865408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765906.488304:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098865408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765906.488309:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765906.488311:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765906.488312:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098865408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765906.488324:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098865536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765906.488334:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765906.488340:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765906/real 1775765906] req@ffff8b2382dae680 x1862022098865536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765906.488347:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098865536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765906.488352:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765906.488354:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765906.488356:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098865536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765907.383737:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765908.407624:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765909.431692:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765910.455583:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765911.479642:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765911.607573:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765911.607584:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765911.607590:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765911.607601:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3373 00000100:00080000:1.0:1775765911.607613:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765911.607629:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765911.607669:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098865664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765911.607698:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765911.607702:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765911.607704:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765911.607709:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3373 00000100:00080000:1.0:1775765911.607713:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765911.607718:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765911.607728:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765911.607742:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098865792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765911.607746:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098865664:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775765911.607754:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765911.607757:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765911.607759:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765911.607763:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3373 00000100:00080000:1.0:1775765911.607766:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765911.607770:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765911.607775:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765911.607784:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098865920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765911.607798:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765911.607802:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765911.607803:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765911.607807:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3373 00000100:00080000:1.0:1775765911.607810:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775765911.607811:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775765911.607813:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765911.607817:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765911.607826:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098866048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765911.607835:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765911.607837:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765911.607839:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765911.607842:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3373 00000100:00080000:1.0:1775765911.607846:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765911.607849:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765911.607849:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765911/real 1775765911] req@ffff8b23864ffb80 x1862022098865664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775765911.607854:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765911.607865:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098866176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765911.607866:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098865664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765911.607884:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765911.607888:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765911.607893:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098865664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765911.607916:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098865792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765911.607941:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765911.607954:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765911/real 1775765911] req@ffff8b23864fe300 x1862022098865792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765911.607968:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098865792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765911.607977:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765911.607981:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765911.607984:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098865792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765911.607991:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098865920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765911.608008:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765911.608015:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765911/real 1775765911] req@ffff8b23864ff100 x1862022098865920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765911.608023:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098865920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765911.608028:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765911.608030:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765911.608032:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098865920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765911.608037:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098866048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765911.608065:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765911.608071:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765911/real 1775765911] req@ffff8b23864fd500 x1862022098866048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765911.608079:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098866048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765911.608084:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765911.608086:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765911.608088:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098866048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765911.608091:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098866176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765911.608101:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765911.608106:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765911/real 1775765911] req@ffff8b23864fdc00 x1862022098866176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765911.608113:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098866176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765911.608118:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765911.608120:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765911.608122:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098866176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765912.503558:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765913.527589:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765914.551561:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765915.575620:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765916.599573:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765916.727620:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765916.727629:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765916.727633:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765916.727641:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3378 00000100:00080000:2.0:1775765916.727651:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765916.727656:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765916.727691:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022098866304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765916.727721:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765916.727725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765916.727726:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765916.727729:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3378 00000100:00080000:2.0:1775765916.727731:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765916.727733:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765916.727740:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765916.727750:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098866432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765916.727763:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765916.727766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765916.727767:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765916.727770:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3378 00000100:00080000:2.0:1775765916.727773:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765916.727775:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765916.727779:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765916.727782:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098866304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765916.727789:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022098866560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765916.727795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765916.727798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765916.727799:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765916.727802:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3378 00000100:00080000:2.0:1775765916.727804:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765916.727806:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765916.727809:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765916.727817:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098866688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765916.727823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765916.727825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765916.727826:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765916.727829:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3378 00000100:00080000:2.0:1775765916.727833:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765916.727835:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765916.727838:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765916.727846:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098866816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765916.727879:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098866432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765916.727907:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098866560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765916.727919:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098866688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765916.727930:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098866816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765917.623761:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765918.647551:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765919.671603:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775765919.799801:0:2363:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:3.0:1775765919.799854:0:2363:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765919.799873:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775765919.799916:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775765919.799928:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775765919.799939:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775765919.799949:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775765919.799957:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765916/real 1775765919] req@ffff8b23b9ea7b80 x1862022098866560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765919.799990:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022098866560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765919.800020:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765919.800026:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765919.800031:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098866560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765919.800040:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765916/real 1775765919] req@ffff8b23b9ea5f80 x1862022098866688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765919.800069:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022098866688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765919.800079:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765919.800088:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765919.800092:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098866688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765919.800100:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765916/real 1775765919] req@ffff8b23b9ea6680 x1862022098866816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765919.800112:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022098866816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765919.800120:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765919.800124:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765919.800128:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098866816:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765919.800135:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765916/real 1775765919] req@ffff8b23b9ea6d80 x1862022098866304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765919.800148:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022098866304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765919.800157:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765919.800161:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765919.800164:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098866304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765919.800172:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765916/real 1775765919] req@ffff8b23b9ea5c00 x1862022098866432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765919.800184:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022098866432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765919.800193:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765919.800197:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765919.800200:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098866432:192.168.202.128@tcp:38:kworker.0 00010000:00010000:2.0:1775765920.695585:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765921.719618:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765921.847582:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765921.847592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765921.847596:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765921.847606:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3383 00000100:00080000:0.0:1775765921.847617:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765921.847624:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765921.847685:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098866944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765921.847722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765921.847726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765921.847728:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765921.847732:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3383 00000100:00080000:0.0:1775765921.847736:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765921.847740:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765921.847751:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765921.847766:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098867072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765921.847777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765921.847781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765921.847783:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765921.847788:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3383 00000100:00080000:0.0:1775765921.847793:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765921.847797:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765921.847805:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765921.847810:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098866944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765921.847824:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098867200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765921.847841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765921.847845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765921.847848:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765921.847852:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3383 00000100:00080000:0.0:1775765921.847858:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765921.847863:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765921.847869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765921.847886:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098867328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765921.847898:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775765921.847900:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765921.847905:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765921.847907:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765921.847912:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3383 00000100:00080000:0.0:1775765921.847920:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765921.847925:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765921.847931:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765921.847945:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765921/real 1775765921] req@ffff8b2382dac380 x1862022098866944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765921.847947:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098867456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765921.847990:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098866944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765921.848528:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765921.848535:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765921.848541:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098866944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765921.848551:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098867072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765921.848584:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765921.848602:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765921/real 1775765921] req@ffff8b2382daf100 x1862022098867072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765921.848617:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098867072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765921.848626:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765921.848630:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765921.848634:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098867072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765921.848670:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098867200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765921.848692:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765921.848704:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765921/real 1775765921] req@ffff8b2382daca80 x1862022098867200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765921.848717:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098867200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765921.848725:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765921.848729:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765921.848732:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098867200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765921.848739:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098867328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765921.848758:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765921.848767:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765921/real 1775765921] req@ffff8b2382dad880 x1862022098867328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765921.848777:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098867328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765921.848785:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765921.848789:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765921.848792:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098867328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765921.848798:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098867456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765921.848817:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765921.848826:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765921/real 1775765921] req@ffff8b2382dac000 x1862022098867456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765921.848837:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098867456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765921.848845:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765921.848849:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765921.848853:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098867456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765922.743541:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765923.767595:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765924.791680:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765925.815581:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765926.839619:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765926.967563:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765926.967589:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765926.967594:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765926.967603:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3388 00000100:00080000:3.0:1775765926.967615:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765926.967621:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765926.967656:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098867584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765926.967688:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765926.967692:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765926.967694:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765926.967697:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3388 00000100:00080000:3.0:1775765926.967701:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765926.967705:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765926.967726:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765926.967737:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098867712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765926.967750:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098867584:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765926.967752:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765926.967763:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765926.967767:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765926.967772:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3388 00000100:00080000:3.0:1775765926.967775:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765926.967778:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765926.967782:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765926.967794:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098867840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765926.967803:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765926.967807:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765926.967808:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765926.967812:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3388 00000100:00080000:3.0:1775765926.967815:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765926.967818:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765926.967822:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765926.967834:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098867968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765926.967842:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775765926.967852:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765926.967856:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765926.967858:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765926.967863:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3388 00000100:00080000:3.0:1775765926.967867:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765926.967871:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765926.967874:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765926.967886:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098868096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765926.967899:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765926/real 1775765926] req@ffff8b23864f1180 x1862022098867584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765926.967920:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098867584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765926.967931:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765926.967935:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765926.967939:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098867584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765926.967960:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098867712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765926.967980:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765926.968009:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765926/real 1775765926] req@ffff8b23864f0700 x1862022098867712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765926.968020:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098867712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765926.968027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765926.968031:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765926.968034:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098867712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765926.968039:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098867840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765926.968072:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765926.968082:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765926/real 1775765926] req@ffff8b23864f0000 x1862022098867840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765926.968100:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098867840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765926.968107:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765926.968110:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765926.968113:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098867840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765926.968118:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098867968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765926.968153:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765926.968161:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765926/real 1775765926] req@ffff8b23864f3480 x1862022098867968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765926.968172:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098867968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765926.968183:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765926.968187:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765926.968192:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098867968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765926.968212:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098868096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765926.968245:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765926.968277:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765926/real 1775765926] req@ffff8b23864f2d80 x1862022098868096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765926.968292:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098868096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765926.968328:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765926.968332:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765926.968335:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098868096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765927.863602:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765928.887611:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765929.911590:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765930.935621:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765931.959624:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765932.087576:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765932.087586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765932.087592:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765932.087602:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3393 00000100:00080000:0.0:1775765932.087613:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765932.087620:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765932.087660:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098868224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765932.087686:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765932.087688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765932.087690:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765932.087693:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3393 00000100:00080000:0.0:1775765932.087697:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765932.087700:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765932.087708:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765932.087717:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098868224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765932.087720:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098868352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765932.087737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765932.087740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765932.087741:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765932.087744:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3393 00000100:00080000:0.0:1775765932.087747:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765932.087750:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765932.087753:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765932.087761:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098868480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765932.087768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765932.087770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765932.087771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765932.087774:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3393 00000100:00080000:0.0:1775765932.087776:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775765932.087778:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775765932.087781:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765932.087786:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765932.087793:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098868608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765932.087799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765932.087802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765932.087803:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765932.087805:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3393 00000100:00080000:0.0:1775765932.087809:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765932.087812:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765932.087816:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765932.087818:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765932/real 1775765932] req@ffff8b2382dac000 x1862022098868224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765932.087827:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098868736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765932.087839:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098868224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765932.087848:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765932.087852:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765932.087856:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098868224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765932.087885:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098868352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765932.087902:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765932.087909:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765932/real 1775765932] req@ffff8b2382dad880 x1862022098868352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765932.087918:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098868352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765932.087923:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765932.087925:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765932.087927:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098868352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765932.087931:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098868480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765932.087942:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765932.087948:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765932/real 1775765932] req@ffff8b2382daca80 x1862022098868480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765932.087955:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098868480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765932.087960:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765932.087961:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765932.087963:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098868480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765932.087967:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098868608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765932.087977:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765932.087982:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765932/real 1775765932] req@ffff8b2382daf100 x1862022098868608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765932.087994:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098868608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765932.087999:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765932.088000:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765932.088002:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098868608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765932.088005:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098868736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765932.088014:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765932.088019:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765932/real 1775765932] req@ffff8b2382dac380 x1862022098868736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765932.088025:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098868736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765932.088030:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765932.088032:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765932.088034:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098868736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765932.983537:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765934.007595:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765935.031568:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765936.055653:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765937.079547:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765937.207567:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765937.207576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765937.207581:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765937.207591:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3398 00000100:00080000:2.0:1775765937.207602:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765937.207609:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765937.207647:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098868864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765937.207677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765937.207682:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765937.207684:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765937.207689:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3398 00000100:00080000:2.0:1775765937.207693:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765937.207697:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765937.207707:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765937.207722:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098868992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765937.207727:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098868864:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775765937.207743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765937.207747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765937.207750:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765937.207755:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3398 00000100:00080000:2.0:1775765937.207760:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765937.207765:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765937.207770:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765937.207783:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098869120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765937.207787:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775765937.207799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765937.207803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765937.207805:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765937.207811:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3398 00000100:00080000:2.0:1775765937.207815:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765937.207819:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765937.207825:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765937.207831:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765937/real 1775765937] req@ffff8b23b9ea5500 x1862022098868864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765937.207840:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098869248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765937.207850:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022098868864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765937.207854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765937.207861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765937.207864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765937.207873:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3398 00000100:00080000:2.0:1775765937.207877:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765937.207877:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765937.207885:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:3.0:1775765937.207886:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765937.207892:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098868864:192.168.202.128@tcp:250:kworker.0 00000008:00100000:2.0:1775765937.207893:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765937.207908:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022098869376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765937.207918:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098868992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765937.207936:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765937.207944:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765937/real 1775765937] req@ffff8b23b9ea4700 x1862022098868992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765937.207952:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022098868992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765937.207957:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765937.207960:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765937.207962:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098868992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765937.207965:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098869120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765937.207975:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765937.207980:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765937/real 1775765937] req@ffff8b23b9ea6a00 x1862022098869120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765937.207987:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022098869120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765937.207992:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765937.207994:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765937.207995:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098869120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765937.207999:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098869248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765937.208009:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765937.208014:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765937/real 1775765937] req@ffff8b23b9ea4000 x1862022098869248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765937.208021:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022098869248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765937.208026:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765937.208028:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765937.208030:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098869248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765937.208039:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098869376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765937.208048:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765937.208052:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765937/real 1775765937] req@ffff8b23b9ea5880 x1862022098869376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765937.208059:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022098869376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765937.208063:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765937.208065:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765937.208067:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098869376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765938.103713:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765939.127610:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765940.151666:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765941.175548:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765942.199759:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765942.327641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765942.327652:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765942.327675:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765942.327688:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3403 00000100:00080000:0.0:1775765942.327703:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765942.327711:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765942.327779:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098869504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765942.327815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765942.327820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765942.327823:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765942.327845:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3403 00000100:00080000:0.0:1775765942.327850:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765942.327854:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765942.327865:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765942.327900:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098869632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765942.327924:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765942.327935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1775765942.327935:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098869504:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775765942.327939:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765942.327955:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3403 00000100:00080000:0.0:1775765942.327962:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765942.327969:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765942.327979:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765942.328006:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098869760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765942.328025:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765942.328031:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765942.328038:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775765942.328058:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775765942.328067:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3403 00000100:00080000:0.0:1775765942.328074:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765942.328080:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765942.328089:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765942.328126:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098869888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765942.328145:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775765942.328152:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765942/real 1775765942] req@ffff8b2382dae680 x1862022098869504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765942.328154:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765942.328212:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765942.328228:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3403 00000100:00080000:0.0:1775765942.328239:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765942.328248:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765942.328257:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098869504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775765942.328260:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775765942.328288:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1775765942.328296:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098870016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765942.328303:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765942.328322:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098869504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765942.328364:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098869632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765942.328410:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765942.328434:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765942/real 1775765942] req@ffff8b2382dad500 x1862022098869632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765942.328538:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098869632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765942.328555:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765942.328561:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765942.328568:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098869632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765942.328597:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098869760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765942.328637:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765942.328677:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765942/real 1775765942] req@ffff8b2382daea00 x1862022098869760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765942.328697:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098869760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765942.328716:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765942.328720:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765942.328724:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098869760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765942.328732:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098869888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765942.328759:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765942.328771:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765942/real 1775765942] req@ffff8b2382daed80 x1862022098869888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765942.328785:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098869888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765942.328794:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765942.328798:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765942.328801:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098869888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765942.328808:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098870016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765942.328843:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765942.328853:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765942/real 1775765942] req@ffff8b2382dadf80 x1862022098870016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765942.328866:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098870016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765942.328875:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765942.328878:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765942.328882:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098870016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765943.223592:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765944.247623:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765945.271600:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765946.295740:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765947.319553:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765947.447595:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765947.447604:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765947.447609:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765947.447619:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3408 00000100:00080000:1.0:1775765947.447632:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765947.447638:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765947.447688:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098870144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765947.447720:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765947.447724:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765947.447726:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765947.447730:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3408 00000100:00080000:1.0:1775765947.447734:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765947.447738:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765947.447748:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765947.447762:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098870272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765947.447775:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775765947.447779:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098870144:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775765947.447780:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765947.447784:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765947.447792:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3408 00000100:00080000:1.0:1775765947.447796:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765947.447800:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765947.447806:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765947.447819:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098870400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765947.447830:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765947.447834:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765947.447837:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765947.447841:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3408 00000100:00080000:1.0:1775765947.447845:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775765947.447850:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775765947.447851:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765947.447858:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765947.447871:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098870528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765947.447881:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765947.447885:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765947.447887:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765947.447891:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3408 00000100:00080000:1.0:1775765947.447895:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765947.447899:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765947.447906:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765947.447906:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765947/real 1775765947] req@ffff8b23864fce00 x1862022098870144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765947.447924:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098870656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765947.447932:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098870144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765947.447947:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765947.447952:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765947.447956:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098870144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765947.447982:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098870272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765947.448004:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765947.448017:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765947/real 1775765947] req@ffff8b23864fe680 x1862022098870272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765947.448029:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098870272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765947.448037:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765947.448040:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765947.448044:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098870272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765947.448049:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098870400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765947.448066:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765947.448076:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765947/real 1775765947] req@ffff8b23864fca80 x1862022098870400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765947.448088:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098870400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765947.448096:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765947.448099:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765947.448102:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098870400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765947.448146:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098870528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765947.448166:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765947.448177:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765947/real 1775765947] req@ffff8b23864fea00 x1862022098870528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765947.448189:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098870528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765947.448197:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765947.448200:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765947.448203:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098870528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765947.448209:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098870656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765947.448225:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765947.448233:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765947/real 1775765947] req@ffff8b23864fd180 x1862022098870656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765947.448243:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098870656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765947.448251:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765947.448255:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765947.448258:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098870656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765948.343702:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765949.367567:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765950.391658:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765951.415572:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765952.439607:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765952.567567:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765952.567577:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765952.567582:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765952.567592:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3413 00000100:00080000:3.0:1775765952.567603:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765952.567610:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765952.567650:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098870784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765952.567712:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765952.567717:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765952.567719:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765952.567738:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3413 00000100:00080000:3.0:1775765952.567743:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765952.567748:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765952.567758:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765952.567771:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098870784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765952.567774:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098870912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765952.567786:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765952.567789:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765952.567791:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765952.567795:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3413 00000100:00080000:3.0:1775765952.567798:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765952.567817:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765952.567821:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765952.567829:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098871040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765952.567836:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765952.567839:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765952.567840:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765952.567844:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3413 00000800:00000100:0.0:1775765952.567848:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775765952.567849:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765952.567853:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765952.567856:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765952.567882:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098871168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765952.567898:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765952.567901:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765952.567902:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765952.567905:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3413 00000100:00080000:3.0:1775765952.567908:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765952.567911:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765952.567915:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765952/real 1775765952] req@ffff8b23864f3100 x1862022098870784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775765952.567916:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765952.567930:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098871296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765952.567937:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098870784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765952.567946:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765952.567950:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765952.567955:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098870784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765952.568002:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098870912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765952.568019:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765952.568028:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765952/real 1775765952] req@ffff8b23864f2a00 x1862022098870912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765952.568090:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098870912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765952.568099:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765952.568102:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765952.568104:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098870912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765952.568108:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098871040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765952.568133:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765952.568140:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765952/real 1775765952] req@ffff8b23864f3b80 x1862022098871040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765952.568148:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098871040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765952.568153:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765952.568155:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765952.568157:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098871040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765952.568160:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098871168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765952.568170:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765952.568175:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765952/real 1775765952] req@ffff8b23864f2d80 x1862022098871168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765952.568182:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098871168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765952.568191:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765952.568193:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765952.568195:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098871168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765952.568199:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098871296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765952.568208:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765952.568213:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765952/real 1775765952] req@ffff8b23864f3480 x1862022098871296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765952.568219:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098871296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765952.568224:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765952.568226:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765952.568228:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098871296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765953.463559:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765954.487649:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765955.511557:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765956.535635:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765957.559608:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765957.687601:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765957.687612:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765957.687617:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765957.687645:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3419 00000100:00080000:3.0:1775765957.687659:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765957.687667:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765957.687733:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098871424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765957.687774:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765957.687777:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765957.687779:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765957.687783:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3419 00000100:00080000:3.0:1775765957.687786:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765957.687789:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765957.687799:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765957.687814:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098871552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765957.687842:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765957.687847:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765957.687850:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765957.687857:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3419 00000100:00100000:0.0:1775765957.687867:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098871424:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765957.687867:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765957.687883:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765957.687890:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765957.687905:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098871680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765957.687933:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765957.687936:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765957.687937:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765957.687941:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3419 00000100:00080000:3.0:1775765957.687945:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765957.687948:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765957.687953:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765957.687954:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765957.687967:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098871808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765957.687976:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765957.687979:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765957.687981:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765957.687984:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3419 00000100:00080000:3.0:1775765957.687987:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765957.687990:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765957.687993:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765957.688001:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765957/real 1775765957] req@ffff8b23864f0000 x1862022098871424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765957.688004:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098871936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765957.688050:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098871424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765957.688061:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765957.688065:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765957.688069:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098871424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765957.688076:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098871552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765957.688106:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765957.688115:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765957/real 1775765957] req@ffff8b23864f0700 x1862022098871552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765957.688123:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098871552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765957.688128:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765957.688130:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765957.688133:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098871552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765957.688153:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098871680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765957.688163:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765957.688169:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765957/real 1775765957] req@ffff8b23864f1180 x1862022098871680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765957.688176:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098871680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765957.688182:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765957.688184:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765957.688186:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098871680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765957.688190:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098871808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765957.688201:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765957.688207:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765957/real 1775765957] req@ffff8b23864f0a80 x1862022098871808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765957.688214:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098871808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765957.688218:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765957.688220:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765957.688222:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098871808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765957.688226:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098871936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765957.688235:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765957.688240:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765957/real 1775765957] req@ffff8b23864f2680 x1862022098871936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765957.688247:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098871936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765957.688251:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765957.688254:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765957.688256:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098871936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765958.583589:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765959.607588:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765960.631617:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765961.655665:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765962.679683:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765962.807595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765962.807603:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765962.807607:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765962.807616:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3424 00000100:00080000:0.0:1775765962.807625:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765962.807631:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765962.807673:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098872064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765962.807697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765962.807699:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765962.807701:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765962.807704:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3424 00000100:00080000:0.0:1775765962.807706:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765962.807709:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765962.807717:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765962.807726:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098872192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765962.807736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775765962.807736:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098872064:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775765962.807742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765962.807747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765962.807752:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3424 00000100:00080000:0.0:1775765962.807756:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765962.807760:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765962.807766:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765962.807780:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098872320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765962.807798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1775765962.807805:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775765962.807806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765962.807810:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765962.807814:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3424 00000100:00080000:0.0:1775765962.807819:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765962.807822:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765962.807827:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765962.807836:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098872448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765962.807843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765962.807846:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765962.807847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765962.807849:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3424 00000100:00080000:0.0:1775765962.807852:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765962.807854:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765962.807857:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765962.807861:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765962/real 1775765962] req@ffff8b2382daf480 x1862022098872064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765962.807871:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098872576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765962.807891:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098872064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765962.807903:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765962.807908:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765962.807912:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098872064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765962.807936:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098872192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765962.808006:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765962.808019:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765962/real 1775765962] req@ffff8b2382dace00 x1862022098872192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765962.808032:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098872192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765962.808041:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765962.808044:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765962.808048:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098872192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765962.808055:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098872320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765962.808072:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765962.808080:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765962/real 1775765962] req@ffff8b2382dafb80 x1862022098872320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765962.808099:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098872320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765962.808108:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765962.808111:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765962.808114:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098872320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765962.808123:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098872448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765962.808137:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765962.808145:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765962/real 1775765962] req@ffff8b2382dac700 x1862022098872448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765962.808157:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098872448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765962.808165:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765962.808168:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765962.808171:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098872448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765962.808176:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098872576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765962.808200:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765962.808209:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765962/real 1775765962] req@ffff8b2382dadf80 x1862022098872576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765962.808220:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098872576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765962.808233:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765962.808236:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765962.808239:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098872576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765963.703588:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765964.727612:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765965.751595:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765966.775615:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765967.799596:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765967.927559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765967.927586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765967.927591:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765967.927600:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3429 00000100:00080000:2.0:1775765967.927612:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765967.927618:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765967.927655:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098872704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765967.927682:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765967.927686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765967.927688:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765967.927692:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3429 00000100:00080000:2.0:1775765967.927697:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765967.927702:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765967.927712:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765967.927721:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098872704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765967.927728:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098872832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765967.927743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765967.927746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765967.927748:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765967.927763:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3429 00000100:00080000:2.0:1775765967.927773:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765967.927778:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765967.927784:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765967.927789:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765967.927802:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098872960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765967.927820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775765967.927824:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765967/real 1775765967] req@ffff8b23b9ea4a80 x1862022098872704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765967.927826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765967.927831:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765967.927840:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3429 00000100:00080000:2.0:1775765967.927845:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775765967.927848:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022098872704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765967.927853:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:3.0:1775765967.927863:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1775765967.927864:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775765967.927869:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765967.927872:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098872704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765967.927881:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098873088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765967.927894:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775765967.927895:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098872832:192.168.202.128@tcp:38:kworker.0 00000100:00080000:2.0:1775765967.927904:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765967.927906:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765967.927927:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3429 00000100:00080000:2.0:1775765967.927932:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765967.927936:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765967.927941:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765967.927956:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098873216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765967.927962:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765967.927980:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765967/real 1775765967] req@ffff8b23b9ea7100 x1862022098872832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765967.927992:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022098872832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765967.927998:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765967.928000:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765967.928003:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098872832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765967.928007:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098872960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765967.928022:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765967.928028:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765967/real 1775765967] req@ffff8b23b9ea4380 x1862022098872960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765967.928035:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022098872960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765967.928041:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765967.928042:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765967.928044:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098872960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765967.928053:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098873088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765967.928064:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765967.928069:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765967/real 1775765967] req@ffff8b23b9ea4e00 x1862022098873088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765967.928076:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022098873088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765967.928081:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765967.928083:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765967.928085:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098873088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765967.928088:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098873216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765967.928097:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765967.928102:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765967/real 1775765967] req@ffff8b23b9ea7480 x1862022098873216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765967.928109:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022098873216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765967.928114:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765967.928116:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765967.928118:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098873216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765968.823587:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765969.847546:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765970.871605:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765971.895650:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765972.919631:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765973.047616:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765973.047623:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765973.047627:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765973.047635:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3434 00000100:00080000:1.0:1775765973.047644:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765973.047649:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775765973.047685:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098873344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765973.047712:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765973.047715:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765973.047717:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765973.047719:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3434 00000100:00080000:1.0:1775765973.047722:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765973.047724:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765973.047731:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765973.047738:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098873472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765973.047746:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765973.047749:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765973.047751:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765973.047756:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3434 00000100:00080000:1.0:1775765973.047758:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765973.047760:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765973.047763:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765973.047770:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098873600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765973.047781:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765973.047784:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765973.047786:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765973.047795:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3434 00000100:00100000:3.0:1775765973.047799:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098873344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775765973.047801:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765973.047813:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765973.047821:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765973.047836:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098873728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765973.047846:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765973.047850:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775765973.047852:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765973.047857:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3434 00000100:00080000:1.0:1775765973.047881:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765973.047885:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775765973.047889:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765973.047902:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098873856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765973.047916:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765973.047984:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765973/real 1775765973] req@ffff8b23864fdf80 x1862022098873344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765973.048008:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098873344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765973.048023:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765973.048028:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765973.048033:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098873344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765973.048041:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098873472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765973.048063:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765973.048075:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765973/real 1775765973] req@ffff8b23864fc380 x1862022098873472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765973.048088:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098873472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765973.048104:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765973.048107:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765973.048111:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098873472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765973.048142:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098873600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765973.048159:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765973.048168:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765973/real 1775765973] req@ffff8b23864fed80 x1862022098873600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765973.048180:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098873600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765973.048189:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765973.048192:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765973.048195:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098873600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765973.048201:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098873728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765973.048218:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765973.048226:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765973/real 1775765973] req@ffff8b23864fc000 x1862022098873728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765973.048239:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098873728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765973.048247:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765973.048250:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765973.048254:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098873728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765973.048259:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098873856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765973.048274:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765973.048281:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765973/real 1775765973] req@ffff8b23864ff800 x1862022098873856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765973.048292:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098873856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765973.048300:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765973.048303:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765973.048306:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098873856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765973.943604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765974.967599:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765975.991583:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765977.015624:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765978.039600:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765978.167609:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765978.167621:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765978.167626:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765978.167637:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3439 00000100:00080000:3.0:1775765978.167650:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765978.167658:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765978.167702:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098873984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765978.167739:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765978.167744:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765978.167746:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765978.167750:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3439 00000100:00080000:3.0:1775765978.167753:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765978.167759:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765978.167770:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765978.167783:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098874112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765978.167798:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765978.167804:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775765978.167807:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098873984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775765978.167808:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765978.167842:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3439 00000100:00080000:3.0:1775765978.167846:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765978.167850:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765978.167855:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765978.167873:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098874240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765978.167884:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765978.167888:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765978.167891:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765978.167895:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3439 00000100:00080000:3.0:1775765978.167908:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765978.167912:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765978.167918:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765978.167920:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765978.167939:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098874368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765978.167952:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765978.167957:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775765978.167959:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765978.167963:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3439 00000100:00080000:3.0:1775765978.167968:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765978.167971:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775765978.167977:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765978.167990:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765978/real 1775765978] req@ffff8b23864f1f80 x1862022098873984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765978.167991:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098874496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765978.168054:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098873984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765978.168069:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765978.168074:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765978.168079:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098873984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775765978.168106:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098874112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765978.168139:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765978.168161:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765978/real 1775765978] req@ffff8b23864f0380 x1862022098874112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765978.168182:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098874112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765978.168197:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765978.168203:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765978.168208:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098874112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765978.168219:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098874240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775765978.168254:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765978.168291:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765978/real 1775765978] req@ffff8b23864f1c00 x1862022098874240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765978.168313:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098874240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765978.168328:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765978.168333:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765978.168339:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098874240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775765978.168350:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098874368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765978.168383:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765978.168396:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765978/real 1775765978] req@ffff8b23864f2680 x1862022098874368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765978.168409:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098874368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765978.168418:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765978.168421:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765978.168425:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098874368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775765978.168431:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098874496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775765978.168449:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765978.168498:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765978/real 1775765978] req@ffff8b23864f0a80 x1862022098874496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765978.168511:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098874496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765978.168520:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765978.168524:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775765978.168527:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098874496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775765979.063643:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765980.087540:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765981.111764:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765982.135697:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765983.159668:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765983.287580:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765983.287592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765983.287597:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765983.287607:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3444 00000100:00080000:0.0:1775765983.287620:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765983.287627:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765983.287676:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098874624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765983.287722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765983.287726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765983.287728:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765983.287731:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3444 00000100:00080000:0.0:1775765983.287734:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765983.287737:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765983.287746:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765983.287780:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098874752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765983.287808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775765983.287812:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098874624:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775765983.287813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765983.287815:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765983.287824:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3444 00000100:00080000:0.0:1775765983.287827:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765983.287830:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765983.287835:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765983.287848:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098874880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765983.287856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765983.287859:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765983.287860:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765983.287863:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3444 00000100:00080000:0.0:1775765983.287866:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765983.287868:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765983.287871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765983.287878:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098875008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765983.287884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765983.287886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765983.287888:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765983.287891:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3444 00000100:00080000:0.0:1775765983.287894:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765983.287896:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765983.287900:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765983.287913:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098875136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765983.287945:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098874752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765983.287986:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098874880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765983.288030:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098875008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765983.288049:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098875136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765984.183631:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765985.207794:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765986.231574:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775765986.359914:0:2364:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775765986.359976:0:2364:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775765986.360006:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765986.360079:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765986.360091:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765986.360108:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775765986.360117:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:3.0:1775765986.360211:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765983/real 1775765986] req@ffff8b2382daea00 x1862022098874752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765986.360250:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098874752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765986.360271:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765986.360278:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765986.360284:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098874752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765986.360295:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765983/real 1775765986] req@ffff8b2382dad500 x1862022098874880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765986.360310:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098874880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765986.360321:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765986.360325:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765986.360329:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098874880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765986.360338:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765983/real 1775765986] req@ffff8b2382dae680 x1862022098875008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765986.360353:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098875008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765986.360363:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765986.360367:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765986.360370:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098875008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765986.360378:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765983/real 1775765986] req@ffff8b2382dac380 x1862022098875136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765986.360390:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098875136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765986.360399:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765986.360403:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765986.360407:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098875136:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765986.360415:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765983/real 1775765986] req@ffff8b2382daed80 x1862022098874624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765986.360429:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098874624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765986.360437:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765986.360442:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765986.360445:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098874624:192.168.202.128@tcp:250:kworker.0 00010000:00010000:3.0:1775765987.255682:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765988.279612:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765988.407592:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765988.407603:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765988.407608:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765988.407619:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3449 00000100:00080000:0.0:1775765988.407632:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765988.407639:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775765988.407691:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098875264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765988.407741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765988.407745:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765988.407747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765988.407751:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3449 00000100:00080000:0.0:1775765988.407756:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765988.407769:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765988.407782:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765988.407809:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098875392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765988.407826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775765988.407827:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098875264:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775765988.407837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765988.407841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765988.407850:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3449 00000100:00080000:0.0:1775765988.407856:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765988.407862:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765988.407869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765988.407887:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098875520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765988.407897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765988.407901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765988.407903:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765988.407908:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3449 00000100:00080000:0.0:1775765988.407914:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765988.407919:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765988.407925:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765988.407937:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775765988.407941:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098875648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765988.407955:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765988.407959:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775765988.407961:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765988.407965:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3449 00000100:00080000:0.0:1775765988.407971:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765988.407977:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775765988.407984:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765988.408002:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098875776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765988.408043:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765988/real 1775765988] req@ffff8b2382daf100 x1862022098875264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765988.408072:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098875264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765988.408084:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765988.408089:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765988.408094:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098875264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765988.408133:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098875392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765988.408165:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765988.408182:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765988/real 1775765988] req@ffff8b2382daca80 x1862022098875392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765988.408203:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098875392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765988.408211:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765988.408215:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765988.408219:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098875392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765988.408226:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098875520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765988.408254:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765988.408276:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765988/real 1775765988] req@ffff8b2382dad880 x1862022098875520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765988.408290:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098875520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765988.408299:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765988.408304:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765988.408307:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098875520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765988.408315:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098875648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765988.408341:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765988.408352:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765988/real 1775765988] req@ffff8b2382dac000 x1862022098875648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765988.408365:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098875648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765988.408520:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765988.408525:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765988.408528:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098875648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765988.408536:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098875776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765988.408561:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765988.408573:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765988/real 1775765988] req@ffff8b2382daed80 x1862022098875776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765988.408586:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098875776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765988.408594:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765988.408598:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765988.408601:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098875776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775765989.303880:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765990.327614:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765991.351599:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765992.375634:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765993.399748:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765993.527596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765993.527609:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765993.527616:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765993.527629:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3454 00000100:00080000:2.0:1775765993.527645:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765993.527655:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765993.527747:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022098875904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765993.527783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765993.527787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765993.527790:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765993.527795:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3454 00000100:00080000:2.0:1775765993.527800:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765993.527807:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765993.527819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765993.527839:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098875904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765993.527840:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098876032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765993.527866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765993.527871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765993.527874:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765993.527883:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3454 00000100:00080000:2.0:1775765993.527889:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765993.527895:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765993.527918:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775765993.527930:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775765993.527937:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098876160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765993.527951:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765993.527954:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765993.527956:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765993.527961:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3454 00000100:00080000:2.0:1775765993.527967:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765993.527974:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765993.527975:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765993/real 1775765993] req@ffff8b23b9ea6300 x1862022098875904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1775765993.527998:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765993.527999:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022098875904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765993.528023:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1775765993.528027:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098876288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765993.528031:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765993.528045:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098875904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775765993.528053:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765993.528056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765993.528058:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765993.528065:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3454 00000100:00080000:2.0:1775765993.528068:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765993.528072:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775765993.528078:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098876032:192.168.202.128@tcp:38:kworker.0 00000008:00100000:2.0:1775765993.528082:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765993.528101:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098876416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775765993.528112:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765993.528132:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765993/real 1775765993] req@ffff8b23b9ea7480 x1862022098876032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765993.528141:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022098876032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765993.528147:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765993.528150:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765993.528152:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098876032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765993.528156:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098876160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765993.528167:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765993.528173:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765993/real 1775765993] req@ffff8b23b9ea4e00 x1862022098876160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765993.528180:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022098876160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765993.528185:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765993.528187:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765993.528189:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098876160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765993.528193:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098876288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765993.528204:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765993.528209:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765993/real 1775765993] req@ffff8b23b9ea4380 x1862022098876288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765993.528216:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022098876288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765993.528221:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765993.528223:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765993.528225:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098876288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765993.528250:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098876416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765993.528271:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765993.528283:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765993/real 1775765993] req@ffff8b23b9ea7100 x1862022098876416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765993.528295:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022098876416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765993.528301:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765993.528303:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765993.528305:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098876416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775765994.423640:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765995.447726:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765996.471741:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765997.495787:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765998.519613:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765998.647644:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765998.647660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765998.647668:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765998.647683:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3460 00000100:00080000:2.0:1775765998.647700:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765998.647710:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775765998.647772:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098876544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765998.647831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765998.647836:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765998.647838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765998.647843:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3460 00000100:00080000:2.0:1775765998.647853:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765998.647858:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765998.647873:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765998.647887:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022098876672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765998.647901:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765998.647906:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765998.647908:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765998.647913:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3460 00000100:00080000:2.0:1775765998.647916:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765998.647921:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765998.647927:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765998.647940:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098876544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775765998.647942:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098876800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765998.647988:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765998.647992:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765998.647995:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765998.648001:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3460 00000100:00080000:2.0:1775765998.648005:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765998.648009:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765998.648015:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765998.648028:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098876928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765998.648038:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765998.648041:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775765998.648047:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775765998.648047:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775765998.648060:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3460 00000100:00080000:2.0:1775765998.648067:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765998.648073:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775765998.648079:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765998.648093:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098877056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775765998.648105:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765998/real 1775765998] req@ffff8b23b9ea4a80 x1862022098876544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765998.648121:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022098876544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765998.648130:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765998.648135:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765998.648138:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098876544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775765998.648144:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098876672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765998.648161:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765998.648169:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765998/real 1775765998] req@ffff8b23b9ea5880 x1862022098876672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765998.648177:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022098876672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765998.648182:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765998.648184:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765998.648186:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098876672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765998.648205:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098876800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775765998.648216:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765998.648222:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765998/real 1775765998] req@ffff8b23b9ea4000 x1862022098876800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765998.648234:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022098876800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765998.648239:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765998.648241:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765998.648243:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098876800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775765998.648247:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098876928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765998.648257:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765998.648263:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765998/real 1775765998] req@ffff8b23b9ea6a00 x1862022098876928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765998.648270:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022098876928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765998.648275:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765998.648276:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765998.648278:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098876928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775765998.648282:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098877056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775765998.648302:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775765998.648307:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765998/real 1775765998] req@ffff8b23b9ea4700 x1862022098877056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765998.648314:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022098877056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765998.648322:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765998.648325:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775765998.648327:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098877056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775765999.543658:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766000.567665:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766001.591739:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766002.615701:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766003.639679:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766003.767621:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766003.767632:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766003.767636:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766003.767644:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3465 00000100:00080000:3.0:1775766003.767654:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766003.767659:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766003.767693:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098877184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766003.767724:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766003.767727:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766003.767729:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766003.767732:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3465 00000100:00080000:3.0:1775766003.767734:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766003.767736:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766003.767745:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766003.767759:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098877312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766003.767780:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766003.767785:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766003.767789:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766003.767793:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098877184:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766003.767800:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3465 00000100:00080000:3.0:1775766003.767806:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766003.767810:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766003.767816:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766003.767828:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098877440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766003.767836:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766003.767838:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766003.767840:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766003.767842:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3465 00000100:00080000:3.0:1775766003.767845:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766003.767847:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766003.767851:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766003.767857:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098877568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766003.767871:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766003.767873:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766003.767875:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766003.767878:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3465 00000100:00080000:3.0:1775766003.767881:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766003.767883:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766003.767887:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766003.767894:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766003.767899:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098877696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766003.767953:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766003/real 1775766003] req@ffff8b23864f1180 x1862022098877184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766003.767975:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098877184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766003.767987:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766003.767992:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766003.767997:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098877184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766003.768028:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098877312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766003.768052:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766003.768063:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766003/real 1775766003] req@ffff8b23864f0700 x1862022098877312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766003.768071:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098877312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766003.768077:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766003.768080:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766003.768082:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098877312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766003.768086:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098877440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766003.768099:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766003.768106:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766003/real 1775766003] req@ffff8b23864f0000 x1862022098877440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766003.768114:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098877440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766003.768119:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766003.768121:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766003.768122:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098877440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766003.768126:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098877568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766003.768138:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766003.768144:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766003/real 1775766003] req@ffff8b23864f3480 x1862022098877568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766003.768152:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098877568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766003.768157:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766003.768159:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766003.768160:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098877568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766003.768164:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098877696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766003.768178:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766003.768183:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766003/real 1775766003] req@ffff8b23864f2d80 x1862022098877696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766003.768192:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098877696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766003.768199:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766003.768202:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766003.768204:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098877696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775766004.663746:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766005.687637:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766006.711660:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766007.735675:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766008.759636:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766008.887554:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766008.887568:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766008.887574:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766008.887586:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3470 00000100:00080000:2.0:1775766008.887599:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766008.887606:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766008.887652:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098877824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766008.887681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766008.887685:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766008.887686:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766008.887689:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3470 00000100:00080000:2.0:1775766008.887692:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766008.887695:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766008.887706:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766008.887716:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098877952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766008.887725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766008.887729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766008.887731:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766008.887734:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3470 00000100:00080000:2.0:1775766008.887736:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766008.887741:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766008.887747:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098877824:192.168.202.128@tcp:250:kworker.0 00000008:00100000:2.0:1775766008.887748:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766008.887762:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022098878080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766008.887771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766008.887773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766008.887774:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766008.887777:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3470 00000100:00080000:2.0:1775766008.887779:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766008.887781:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766008.887784:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766008.887791:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098878208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766008.887797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766008.887800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766008.887802:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766008.887805:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3470 00000100:00080000:2.0:1775766008.887808:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766008.887810:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766008.887813:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766008.887820:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098878336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766008.887834:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766008.887875:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766008/real 1775766008] req@ffff8b23b9ea5500 x1862022098877824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766008.887890:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022098877824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766008.887899:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766008.887902:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766008.887905:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098877824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766008.887911:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098877952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766008.887929:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766008.887937:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766008/real 1775766008] req@ffff8b23b9ea5c00 x1862022098877952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766008.887944:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022098877952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766008.887949:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766008.887951:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766008.887953:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098877952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766008.887976:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098878080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766008.887996:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766008.888008:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766008/real 1775766008] req@ffff8b23b9ea6d80 x1862022098878080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766008.888020:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022098878080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766008.888027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766008.888030:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766008.888033:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098878080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766008.888039:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098878208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766008.888060:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766008.888071:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766008/real 1775766008] req@ffff8b23b9ea6680 x1862022098878208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766008.888083:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022098878208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766008.888091:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766008.888094:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766008.888097:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098878208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766008.888104:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098878336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766008.888121:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766008.888129:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766008/real 1775766008] req@ffff8b23b9ea5f80 x1862022098878336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766008.888137:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022098878336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766008.888142:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766008.888144:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766008.888146:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098878336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766009.783605:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766010.807581:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766011.831618:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766012.855636:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766013.879602:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766014.007559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766014.007567:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766014.007570:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766014.007578:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3475 00000100:00080000:2.0:1775766014.007588:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766014.007595:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766014.007635:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022098878464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766014.007675:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766014.007678:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766014.007680:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766014.007682:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3475 00000100:00080000:2.0:1775766014.007686:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766014.007689:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766014.007697:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766014.007708:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098878592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766014.007718:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766014.007720:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098878464:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775766014.007722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766014.007725:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766014.007730:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3475 00000100:00080000:2.0:1775766014.007732:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766014.007734:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766014.007738:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766014.007747:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098878720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766014.007754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766014.007756:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766014.007758:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766014.007760:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3475 00000100:00080000:2.0:1775766014.007762:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766014.007764:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766014.007768:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766014.007775:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022098878848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766014.007781:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775766014.007787:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766014.007790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766014.007792:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766014.007802:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3475 00000100:00080000:2.0:1775766014.007805:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766014.007808:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766014.007811:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766014.007819:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766014/real 1775766014] req@ffff8b23b9ea7b80 x1862022098878464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766014.007822:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098878976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766014.007854:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022098878464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766014.007862:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766014.007866:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766014.007869:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098878464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766014.007889:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098878592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766014.007904:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766014.007912:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766014/real 1775766014] req@ffff8b23b9ea5f80 x1862022098878592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766014.007920:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022098878592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766014.007929:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766014.007931:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766014.007933:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098878592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766014.007937:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098878720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766014.007947:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766014.007952:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766014/real 1775766014] req@ffff8b23b9ea6680 x1862022098878720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766014.007959:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022098878720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766014.007964:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766014.007966:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766014.007968:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098878720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766014.007971:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098878848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766014.007981:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766014.007986:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766014/real 1775766014] req@ffff8b23b9ea6d80 x1862022098878848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766014.007994:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022098878848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766014.007998:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766014.008000:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766014.008002:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098878848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766014.008006:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098878976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766014.008017:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766014.008022:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766014/real 1775766014] req@ffff8b23b9ea5c00 x1862022098878976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766014.008029:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022098878976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766014.008034:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766014.008035:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766014.008037:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098878976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775766014.903652:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766015.927713:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766016.951635:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766017.975673:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766018.999620:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766019.127623:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766019.127634:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766019.127639:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766019.127650:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3480 00000100:00080000:0.0:1775766019.127664:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766019.127670:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766019.127719:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098879104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766019.127750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766019.127753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766019.127754:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766019.127757:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3480 00000100:00080000:0.0:1775766019.127759:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766019.127763:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766019.127772:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766019.127780:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098879232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766019.127789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766019.127792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766019.127793:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766019.127796:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3480 00000100:00080000:0.0:1775766019.127798:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766019.127802:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766019.127805:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766019.127815:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098879360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766019.127822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766019.127825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766019.127826:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766019.127829:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3480 00000100:00080000:0.0:1775766019.127830:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766019.127832:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766019.127835:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766019.127841:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098879488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766019.127848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766019.127851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766019.127853:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766019.127858:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3480 00000100:00080000:0.0:1775766019.127863:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766019.127865:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098879104:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766019.127867:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766019.127873:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766019.127888:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098879616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766019.127960:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766019.128007:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766019/real 1775766019] req@ffff8b2382dac380 x1862022098879104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766019.128028:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098879104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766019.128040:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766019.128045:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766019.128050:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098879104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766019.128058:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098879232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766019.128084:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766019.128098:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766019/real 1775766019] req@ffff8b2382dae680 x1862022098879232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766019.128112:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098879232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766019.128121:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766019.128125:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766019.128128:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098879232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766019.128135:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098879360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766019.128157:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766019.128169:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766019/real 1775766019] req@ffff8b2382dad500 x1862022098879360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766019.128182:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098879360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766019.128191:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766019.128194:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766019.128198:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098879360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766019.128228:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098879488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766019.128249:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766019.128261:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766019/real 1775766019] req@ffff8b2382daea00 x1862022098879488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766019.128273:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098879488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766019.128287:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766019.128290:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766019.128294:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098879488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766019.128300:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098879616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766019.128320:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766019.128330:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766019/real 1775766019] req@ffff8b2382dadf80 x1862022098879616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766019.128342:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098879616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766019.128351:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766019.128355:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766019.128358:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098879616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766020.023707:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766021.047602:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766022.071742:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766023.095659:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766024.119678:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766024.247602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766024.247611:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766024.247616:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766024.247624:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3485 00000100:00080000:0.0:1775766024.247634:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766024.247640:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766024.247674:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098879744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766024.247699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766024.247702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766024.247704:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766024.247706:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3485 00000100:00080000:0.0:1775766024.247709:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766024.247711:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766024.247718:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766024.247727:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098879872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766024.247737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766024.247740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766024.247741:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766024.247746:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3485 00000100:00080000:0.0:1775766024.247752:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766024.247754:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098879744:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766024.247757:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766024.247767:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766024.247779:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098880000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766024.247786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766024.247788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766024.247791:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766024.247795:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3485 00000100:00080000:0.0:1775766024.247797:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766024.247799:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766024.247802:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766024.247812:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098880128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766024.247819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766024.247822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766024.247823:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766024.247825:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3485 00000100:00080000:0.0:1775766024.247828:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766024.247830:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766024.247834:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766024.247840:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098880256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766024.247867:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766024.247921:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766024/real 1775766024] req@ffff8b2382dac700 x1862022098879744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766024.247940:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098879744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766024.247957:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766024.247961:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766024.247965:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098879744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766024.247997:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098879872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766024.248016:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766024.248036:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766024/real 1775766024] req@ffff8b2382dafb80 x1862022098879872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766024.248046:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098879872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766024.248053:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766024.248055:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766024.248058:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098879872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766024.248063:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098880000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766024.248078:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766024.248085:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766024/real 1775766024] req@ffff8b2382dace00 x1862022098880000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766024.248112:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098880000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766024.248119:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766024.248121:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766024.248123:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098880000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766024.248129:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098880128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766024.248146:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766024.248153:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766024/real 1775766024] req@ffff8b2382daf480 x1862022098880128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766024.248163:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098880128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766024.248169:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766024.248171:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766024.248174:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098880128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766024.248178:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098880256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766024.248191:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766024.248197:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766024/real 1775766024] req@ffff8b2382dadf80 x1862022098880256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766024.248205:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098880256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766024.248211:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766024.248213:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766024.248215:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098880256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775766025.143545:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766026.167627:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766027.191594:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766028.215645:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766029.239623:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766029.367609:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766029.367618:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766029.367622:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766029.367632:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3490 00000100:00080000:0.0:1775766029.367643:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766029.367651:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766029.367702:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098880384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766029.367726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766029.367729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766029.367731:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766029.367733:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3490 00000100:00080000:0.0:1775766029.367736:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766029.367739:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766029.367746:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766029.367755:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098880512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766029.367764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766029.367769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766029.367771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766029.367774:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3490 00000100:00080000:0.0:1775766029.367777:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766029.367779:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766029.367782:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766029.367791:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098880640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766029.367798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766029.367801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766029.367803:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766029.367806:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3490 00000100:00080000:0.0:1775766029.367809:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766029.367812:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766029.367815:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766029.367825:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098880384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766029.367827:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098880768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766029.367851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766029.367853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766029.367854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766029.367856:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3490 00000100:00080000:0.0:1775766029.367858:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766029.367861:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766029.367865:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766029.367874:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098880896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766029.367924:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766029.367968:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766029/real 1775766029] req@ffff8b2382daea00 x1862022098880384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766029.367986:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098880384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766029.367995:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766029.367999:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766029.368002:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098880384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766029.368011:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098880512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766029.368040:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766029.368056:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766029/real 1775766029] req@ffff8b2382dad500 x1862022098880512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766029.368073:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098880512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766029.368083:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766029.368086:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766029.368089:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098880512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766029.368094:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098880640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766029.368118:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766029.368131:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766029/real 1775766029] req@ffff8b2382dae680 x1862022098880640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766029.368155:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098880640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766029.368166:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766029.368170:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766029.368174:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098880640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766029.368202:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098880768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766029.368236:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766029.368250:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766029/real 1775766029] req@ffff8b2382dac380 x1862022098880768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766029.368266:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098880768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766029.368276:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766029.368280:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766029.368284:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098880768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766029.368290:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098880896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766029.368309:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766029.368317:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766029/real 1775766029] req@ffff8b2382daed80 x1862022098880896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766029.368348:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098880896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766029.368360:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766029.368363:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766029.368367:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098880896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766030.263644:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766031.287633:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766032.311690:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766033.335590:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766034.359651:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766034.487586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766034.487599:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766034.487606:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766034.487619:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3495 00000100:00080000:0.0:1775766034.487636:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766034.487646:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766034.487695:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098881024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766034.487743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766034.487757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766034.487760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766034.487767:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3495 00000100:00080000:0.0:1775766034.487774:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766034.487781:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766034.487796:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766034.487812:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098881024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766034.487824:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098881152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766034.487846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766034.487852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766034.487855:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766034.487863:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3495 00000100:00080000:0.0:1775766034.487871:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766034.487878:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766034.487887:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766034.487914:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098881280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766034.487914:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766034.487936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766034.487942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766034.487945:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766034.487951:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3495 00000100:00100000:3.0:1775766034.487972:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766034/real 1775766034] req@ffff8b2382dac000 x1862022098881024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766034.487977:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766034.487991:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766034.488001:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766034.488006:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098881024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766034.488021:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766034.488029:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766034.488030:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098881408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766034.488037:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098881024:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766034.488063:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766034.488069:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766034.488072:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766034.488079:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098881152:192.168.202.128@tcp:38:kworker.0 00000100:00080000:0.0:1775766034.488083:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3495 00000100:00080000:0.0:1775766034.488094:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766034.488102:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766034.488112:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766034.488121:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766034.488140:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098881536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766034.488143:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766034/real 1775766034] req@ffff8b2382dad880 x1862022098881152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766034.488193:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098881152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766034.488226:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766034.488233:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766034.488240:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098881152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766034.488251:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098881280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766034.488331:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766034.488355:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766034/real 1775766034] req@ffff8b2382daca80 x1862022098881280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766034.488371:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098881280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766034.488382:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766034.488386:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766034.488390:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098881280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766034.488398:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098881408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766034.488424:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766034.488435:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766034/real 1775766034] req@ffff8b2382daf100 x1862022098881408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766034.488449:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098881408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766034.488458:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766034.488462:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766034.488537:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098881408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766034.488577:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098881536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766034.488634:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766034.488649:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766034/real 1775766034] req@ffff8b2382dac000 x1862022098881536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766034.488662:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098881536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766034.488671:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766034.488674:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766034.488678:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098881536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775766035.383574:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766036.407666:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766037.431596:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766038.455618:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766039.479578:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766039.607608:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766039.607617:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766039.607621:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766039.607631:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3501 00000100:00080000:1.0:1775766039.607643:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766039.607649:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766039.607698:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098881664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766039.607724:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766039.607729:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766039.607731:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766039.607734:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3501 00000100:00080000:1.0:1775766039.607737:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766039.607740:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766039.607749:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766039.607760:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098881792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766039.607769:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098881664:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766039.607772:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766039.607777:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766039.607779:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766039.607783:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3501 00000100:00080000:1.0:1775766039.607786:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766039.607789:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766039.607792:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766039.607803:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098881920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766039.607810:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766039.607813:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766039.607814:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766039.607817:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3501 00000100:00080000:1.0:1775766039.607821:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766039.607833:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766039.607838:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766039.607842:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766039.607850:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098882048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766039.607859:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766039.607862:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766039.607863:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766039.607866:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3501 00000100:00080000:1.0:1775766039.607868:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766039.607871:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766039.607875:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766039.607880:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766039/real 1775766039] req@ffff8b23864ff800 x1862022098881664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766039.607889:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098882176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766039.607908:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098881664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766039.607918:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766039.607923:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766039.607927:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098881664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766039.607962:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098881792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766039.607991:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766039.608006:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766039/real 1775766039] req@ffff8b23864fc000 x1862022098881792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766039.608038:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098881792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766039.608048:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766039.608051:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766039.608054:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098881792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766039.608060:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098881920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766039.608079:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766039.608087:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766039/real 1775766039] req@ffff8b23864fed80 x1862022098881920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766039.608098:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098881920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766039.608109:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766039.608114:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766039.608117:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098881920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766039.608125:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098882048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766039.608150:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766039.608164:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766039/real 1775766039] req@ffff8b23864fc380 x1862022098882048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766039.608186:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098882048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766039.608212:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766039.608215:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766039.608219:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098882048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766039.608242:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098882176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766039.608277:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766039.608289:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766039/real 1775766039] req@ffff8b23864fdf80 x1862022098882176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766039.608299:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098882176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766039.608306:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766039.608308:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766039.608311:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098882176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766040.503635:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766041.527538:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766042.551606:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766043.575535:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766044.599747:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766044.727606:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766044.727617:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766044.727630:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766044.727645:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3506 00000100:00080000:1.0:1775766044.727676:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766044.727685:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766044.727741:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098882304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766044.727771:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766044.727774:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766044.727776:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766044.727779:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3506 00000100:00080000:1.0:1775766044.727795:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766044.727799:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766044.727810:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766044.727820:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098882432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766044.727830:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766044.727832:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766044.727834:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766044.727838:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3506 00000100:00080000:1.0:1775766044.727841:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766044.727843:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098882304:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766044.727845:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766044.727860:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766044.727869:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098882560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766044.727876:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766044.727879:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766044.727880:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766044.727883:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3506 00000100:00080000:1.0:1775766044.727885:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766044.727887:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766044.727891:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766044.727900:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098882688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766044.727911:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766044.727914:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766044.727915:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766044.727917:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3506 00000100:00080000:1.0:1775766044.727923:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766044.727925:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766044.727928:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766044.727935:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766044.727937:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098882816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766044.727985:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766044/real 1775766044] req@ffff8b23864fd180 x1862022098882304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766044.728006:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098882304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766044.728014:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766044.728017:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766044.728020:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098882304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766044.728026:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098882432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766044.728042:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766044.728050:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766044/real 1775766044] req@ffff8b23864fea00 x1862022098882432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766044.728058:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098882432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766044.728072:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766044.728074:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766044.728076:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098882432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766044.728101:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098882560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766044.728114:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766044.728127:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766044/real 1775766044] req@ffff8b23864fca80 x1862022098882560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766044.728135:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098882560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766044.728145:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766044.728147:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766044.728150:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098882560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766044.728153:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098882688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766044.728164:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766044.728169:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766044/real 1775766044] req@ffff8b23864fe680 x1862022098882688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766044.728176:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098882688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766044.728194:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766044.728196:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766044.728198:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098882688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766044.728202:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098882816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766044.728214:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766044.728221:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766044/real 1775766044] req@ffff8b23864fce00 x1862022098882816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766044.728228:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098882816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766044.728233:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766044.728235:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766044.728238:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098882816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775766045.623617:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766046.647587:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766047.671583:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766048.695611:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766049.719600:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766049.847561:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766049.847569:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766049.847574:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766049.847588:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3511 00000100:00080000:3.0:1775766049.847598:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766049.847604:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766049.847648:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098882944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766049.847671:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766049.847674:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766049.847675:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766049.847678:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3511 00000100:00080000:3.0:1775766049.847681:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766049.847684:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766049.847692:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766049.847703:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098883072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766049.847704:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098882944:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766049.847729:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766049.847732:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766049.847733:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766049.847736:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3511 00000100:00080000:3.0:1775766049.847740:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766049.847749:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766049.847755:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766049.847768:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098883200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766049.847778:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766049.847782:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766049.847784:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766049.847789:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3511 00000100:00080000:3.0:1775766049.847795:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766049.847800:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766049.847807:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766049.847814:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098883072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766049.847832:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098883328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766049.847845:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766049.847849:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1775766049.847852:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098883200:192.168.202.128@tcp:38:kworker.0 00000100:00080000:3.0:1775766049.847853:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766049.847862:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3511 00000100:00080000:3.0:1775766049.847868:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766049.847873:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766049.847878:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098883328:192.168.202.128@tcp:8:kworker.0 00000008:00100000:3.0:1775766049.847880:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766049.847905:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098883456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766049.847949:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098883456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766050.743617:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766051.767549:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766052.791662:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775766052.919693:0:2365:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775766052.919761:0:2365:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775766052.919806:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766052.919841:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766052.919850:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766052.919859:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766052.919878:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775766052.919967:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766049/real 1775766052] req@ffff8b23864f3480 x1862022098883456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766052.920011:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098883456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766052.920027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766052.920035:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766052.920040:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098883456:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766052.920051:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766049/real 1775766052] req@ffff8b23864f2d80 x1862022098883328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766052.920066:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098883328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766052.920076:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766052.920081:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766052.920085:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098883328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766052.920130:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766049/real 1775766052] req@ffff8b23864f2a00 x1862022098883072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766052.920147:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098883072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766052.920156:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766052.920160:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766052.920165:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098883072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766052.920175:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766049/real 1775766052] req@ffff8b23864f3100 x1862022098883200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766052.920190:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098883200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766052.920199:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766052.920204:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766052.920208:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098883200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766052.920217:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766049/real 1775766052] req@ffff8b23864f3b80 x1862022098882944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766052.920231:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098882944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766052.920240:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766052.920244:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766052.920248:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098882944:192.168.202.128@tcp:250:kworker.0 00010000:00010000:2.0:1775766053.815573:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766054.839723:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766054.967593:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766054.967602:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766054.967608:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766054.967617:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3516 00000100:00080000:2.0:1775766054.967627:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766054.967634:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766054.967669:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098883584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766054.967698:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766054.967702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766054.967704:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766054.967709:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3516 00000100:00080000:2.0:1775766054.967714:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766054.967719:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766054.967730:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766054.967747:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098883712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766054.967795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766054.967799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766054.967802:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766054.967807:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3516 00000100:00080000:2.0:1775766054.967811:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766054.967821:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098883584:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775766054.967821:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766054.967841:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766054.967859:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098883840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766054.967878:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766054.967882:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766054.967884:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766054.967889:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3516 00000100:00080000:2.0:1775766054.967894:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766054.967899:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766054.967908:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766054.967923:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775766054.967928:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098883968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766054.967942:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766054.967947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766054.967950:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766054.967957:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3516 00000100:00080000:2.0:1775766054.967963:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766054.967967:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766054.967972:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766054.967985:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022098884096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766054.968004:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766054/real 1775766054] req@ffff8b23b9ea5500 x1862022098883584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766054.968035:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022098883584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766054.968050:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766054.968056:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766054.968061:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098883584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766054.968071:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098883712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766054.968121:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766054.968138:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766054/real 1775766054] req@ffff8b23b9ea4700 x1862022098883712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766054.968151:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022098883712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766054.968160:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766054.968163:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766054.968167:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098883712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766054.968202:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098883840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766054.968225:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766054.968236:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766054/real 1775766054] req@ffff8b23b9ea6a00 x1862022098883840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766054.968249:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022098883840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766054.968257:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766054.968261:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766054.968264:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098883840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766054.968271:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098883968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766054.968302:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766054.968314:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766054/real 1775766054] req@ffff8b23b9ea4000 x1862022098883968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766054.968327:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022098883968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766054.968335:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766054.968339:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766054.968342:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098883968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766054.968349:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098884096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766054.968372:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766054.968385:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766054/real 1775766054] req@ffff8b23b9ea5880 x1862022098884096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766054.968405:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022098884096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766054.968422:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766054.968426:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766054.968430:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098884096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775766055.863577:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766056.887595:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766057.911553:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766058.935610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766059.959548:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766060.087593:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766060.087600:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766060.087604:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766060.087612:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3521 00000100:00080000:1.0:1775766060.087621:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766060.087626:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766060.087660:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098884224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766060.087682:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766060.087685:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766060.087687:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766060.087689:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3521 00000100:00080000:1.0:1775766060.087691:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766060.087694:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766060.087702:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766060.087714:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098884352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766060.087726:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766060.087730:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766060.087732:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766060.087737:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3521 00000100:00080000:1.0:1775766060.087741:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766060.087746:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766060.087751:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098884224:192.168.202.128@tcp:250:kworker.0 00000008:00100000:1.0:1775766060.087754:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766060.087770:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098884480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766060.087782:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766060.087786:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766060.087788:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766060.087792:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3521 00000100:00080000:1.0:1775766060.087807:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775766060.087812:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775766060.087816:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766060.087824:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766060.087838:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098884608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766060.087845:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766060.087847:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766060.087849:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766060.087852:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3521 00000100:00080000:1.0:1775766060.087856:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766060.087862:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766060.087863:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766060/real 1775766060] req@ffff8b23864fdc00 x1862022098884224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775766060.087866:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766060.087880:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098884736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766060.087887:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098884224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766060.087896:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766060.087899:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766060.087903:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098884224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766060.087910:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098884352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766060.087928:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766060.087936:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766060/real 1775766060] req@ffff8b23864fd500 x1862022098884352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766060.087944:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098884352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766060.087949:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766060.087952:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766060.087954:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098884352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766060.087972:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098884480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766060.087983:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766060.087989:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766060/real 1775766060] req@ffff8b23864ff100 x1862022098884480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766060.087996:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098884480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766060.088001:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766060.088003:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766060.088006:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098884480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766060.088009:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098884608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766060.088019:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766060.088024:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766060/real 1775766060] req@ffff8b23864fe300 x1862022098884608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766060.088047:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098884608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766060.088053:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766060.088055:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766060.088057:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098884608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766060.088062:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098884736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766060.088076:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766060.088083:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766060/real 1775766060] req@ffff8b23864ffb80 x1862022098884736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766060.088090:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098884736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766060.088095:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766060.088097:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766060.088099:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098884736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766060.983568:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766062.007542:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766063.031681:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775766064.055579:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766065.079648:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766065.207538:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766065.207552:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766065.207556:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766065.207565:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3526 00000100:00080000:1.0:1775766065.207575:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766065.207581:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766065.207614:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098884864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766065.207638:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766065.207643:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766065.207645:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766065.207647:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3526 00000100:00080000:1.0:1775766065.207650:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766065.207652:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766065.207660:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766065.207675:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098884992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766065.207699:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766065.207703:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766065.207705:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766065.207713:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3526 00000100:00100000:0.0:1775766065.207714:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098884864:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766065.207717:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766065.207724:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766065.207729:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766065.207742:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098885120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766065.207750:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766065.207753:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766065.207754:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766065.207756:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3526 00000100:00080000:1.0:1775766065.207767:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766065.207771:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775766065.207775:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:1.0:1775766065.207779:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766065.207791:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098885248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766065.207798:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766065.207801:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766065.207802:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766065.207805:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3526 00000100:00080000:1.0:1775766065.207808:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766065.207811:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766065.207815:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766065/real 1775766065] req@ffff8b23864ffb80 x1862022098884864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775766065.207816:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766065.207830:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098885376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766065.207836:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098884864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766065.207849:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766065.207853:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766065.207856:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098884864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766065.207862:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098884992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766065.207877:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766065.207884:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766065/real 1775766065] req@ffff8b23864fe300 x1862022098884992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766065.207891:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098884992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766065.207901:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766065.207903:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766065.207906:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098884992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766065.207925:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098885120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766065.207935:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766065.207941:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766065/real 1775766065] req@ffff8b23864ff100 x1862022098885120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766065.207948:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098885120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766065.207953:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766065.207955:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766065.207957:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098885120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766065.207961:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098885248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766065.207970:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766065.207976:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766065/real 1775766065] req@ffff8b23864fd500 x1862022098885248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766065.208009:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098885248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766065.208015:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766065.208017:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766065.208019:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098885248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766065.208024:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098885376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766065.208039:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766065.208046:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766065/real 1775766065] req@ffff8b23864fdc00 x1862022098885376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766065.208053:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098885376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766065.208058:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766065.208060:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766065.208062:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098885376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:2.0:1775766066.103536:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766067.127748:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766068.151630:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766069.175714:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766070.199603:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766070.327576:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766070.327588:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766070.327592:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766070.327604:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3531 00000100:00080000:0.0:1775766070.327625:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766070.327631:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766070.327665:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098885504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766070.327691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766070.327695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766070.327702:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766070.327706:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3531 00000100:00080000:0.0:1775766070.327710:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766070.327715:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766070.327725:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766070.327738:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098885632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766070.327743:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098885504:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766070.327750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766070.327754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766070.327756:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766070.327759:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3531 00000100:00080000:0.0:1775766070.327763:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766070.327767:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766070.327771:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766070.327781:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098885760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766070.327790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766070.327793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766070.327795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766070.327798:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3531 00000100:00080000:0.0:1775766070.327801:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766070.327805:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775766070.327805:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:0.0:1775766070.327809:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766070.327820:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098885888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766070.327829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766070.327832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766070.327834:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766070.327837:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3531 00000100:00080000:0.0:1775766070.327840:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766070.327844:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766070.327845:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766070/real 1775766070] req@ffff8b2382daed80 x1862022098885504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775766070.327849:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766070.327858:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098886016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766070.327863:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098885504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766070.327873:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766070.327877:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766070.327881:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098885504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766070.327909:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098885632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766070.327952:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766070.327963:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766070/real 1775766070] req@ffff8b2382dac380 x1862022098885632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766070.327972:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098885632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766070.327978:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766070.327980:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766070.327983:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098885632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766070.327987:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098885760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766070.328000:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766070.328006:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766070/real 1775766070] req@ffff8b2382dae680 x1862022098885760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766070.328014:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098885760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766070.328019:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766070.328021:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766070.328023:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098885760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766070.328027:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098885888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766070.328038:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766070.328043:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766070/real 1775766070] req@ffff8b2382dad500 x1862022098885888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766070.328050:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098885888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766070.328055:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766070.328057:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766070.328060:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098885888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766070.328063:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098886016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766070.328073:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766070.328078:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766070/real 1775766070] req@ffff8b2382daea00 x1862022098886016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766070.328085:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098886016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766070.328090:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766070.328092:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766070.328094:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098886016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766071.223651:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766072.247690:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766073.271626:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766074.295724:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766075.319679:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766075.447760:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766075.447775:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766075.447783:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766075.447814:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3536 00000100:00080000:3.0:1775766075.447833:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766075.447843:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766075.447921:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098886144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766075.447969:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766075.447976:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766075.447980:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766075.447988:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3536 00000100:00080000:3.0:1775766075.447995:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766075.448004:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766075.448020:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766075.448041:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098886144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766075.448050:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098886272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766075.448077:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766075.448083:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766075.448087:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766075.448095:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3536 00000100:00080000:3.0:1775766075.448102:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766075.448110:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766075.448120:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766075.448122:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766075.448150:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098886400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766075.448172:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766075.448179:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766075/real 1775766075] req@ffff8b23864f0000 x1862022098886144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766075.448183:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766075.448191:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766075.448209:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3536 00000100:00100000:0.0:1775766075.448216:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098886144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766075.448236:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766075.448238:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766075.448243:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766075.448253:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098886144:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766075.448255:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766075.448271:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766075.448288:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098886272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766075.448301:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098886528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766075.448320:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766075.448326:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766075.448335:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766075.448341:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766075.448342:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766075/real 1775766075] req@ffff8b23864f0700 x1862022098886272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766075.448357:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3536 00000100:00080000:3.0:1775766075.448371:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766075.448381:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766075.448387:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098886272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775766075.448394:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775766075.448434:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766075.448447:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766075.448449:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098886656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766075.448460:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098886272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766075.448517:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098886400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766075.448572:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766075.448589:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766075/real 1775766075] req@ffff8b23864f1180 x1862022098886400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766075.448605:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098886400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766075.448615:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766075.448619:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766075.448622:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098886400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766075.448639:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098886528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766075.448657:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766075.448667:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766075/real 1775766075] req@ffff8b23864f0a80 x1862022098886528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766075.448700:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098886528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766075.448713:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766075.448718:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766075.448725:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098886528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766075.448736:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098886656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766075.448756:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766075.448769:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766075/real 1775766075] req@ffff8b23864f2680 x1862022098886656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766075.448780:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098886656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766075.448789:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766075.448792:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766075.448796:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098886656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766076.343748:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766077.367662:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766078.391710:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766079.415615:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766080.439632:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766080.567572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766080.567580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766080.567584:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766080.567592:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3541 00000100:00080000:2.0:1775766080.567601:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766080.567606:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766080.567640:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098886784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766080.567661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766080.567664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766080.567665:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766080.567668:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3541 00000100:00080000:2.0:1775766080.567670:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766080.567672:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766080.567679:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766080.567687:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098886912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766080.567694:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766080.567696:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766080.567697:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766080.567700:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3541 00000100:00080000:2.0:1775766080.567701:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766080.567704:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766080.567707:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766080.567712:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098887040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766080.567718:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766080.567720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766080.567722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766080.567725:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3541 00000100:00080000:2.0:1775766080.567727:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766080.567730:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766080.567733:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766080.567741:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098887168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766080.567745:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098886784:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775766080.567760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766080.567763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766080.567775:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766080.567781:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3541 00000100:00080000:2.0:1775766080.567785:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766080.567789:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766080.567795:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766080.567809:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098887296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766080.567860:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766080.567916:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766080/real 1775766080] req@ffff8b23b9ea4a80 x1862022098886784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766080.567930:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022098886784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766080.567938:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766080.567941:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766080.567943:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098886784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766080.567949:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098886912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766080.567964:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766080.567973:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766080/real 1775766080] req@ffff8b23b9ea7100 x1862022098886912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766080.567981:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022098886912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766080.567986:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766080.567988:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766080.567990:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098886912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766080.567994:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098887040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766080.568005:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766080.568011:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766080/real 1775766080] req@ffff8b23b9ea4380 x1862022098887040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766080.568018:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022098887040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766080.568023:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766080.568025:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766080.568026:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098887040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766080.568045:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098887168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766080.568056:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766080.568062:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766080/real 1775766080] req@ffff8b23b9ea4e00 x1862022098887168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766080.568068:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022098887168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766080.568074:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766080.568076:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766080.568078:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098887168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766080.568081:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098887296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766080.568104:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766080.568113:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766080/real 1775766080] req@ffff8b23b9ea7480 x1862022098887296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766080.568124:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022098887296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766080.568151:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766080.568155:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766080.568158:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098887296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766081.463604:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766082.487582:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766083.511659:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766084.535635:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766085.559662:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766085.687657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766085.687668:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766085.687673:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766085.687684:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3547 00000100:00080000:0.0:1775766085.687712:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766085.687718:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766085.687764:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098887424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766085.687836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766085.687840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766085.687841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766085.687844:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3547 00000100:00080000:0.0:1775766085.687848:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766085.687851:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766085.687861:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766085.687875:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098887552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766085.687883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766085.687886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766085.687888:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766085.687891:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3547 00000100:00080000:0.0:1775766085.687894:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766085.687897:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766085.687899:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766085.687910:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098887680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766085.687917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766085.687921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766085.687922:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766085.687928:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098887424:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766085.687929:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3547 00000100:00080000:0.0:1775766085.687934:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766085.687939:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766085.687942:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766085.687958:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098887808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766085.687964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766085.687966:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766085.687968:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766085.687970:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3547 00000100:00080000:0.0:1775766085.687972:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766085.687974:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766085.687977:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766085.687985:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098887936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766085.688009:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766085.688052:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766085/real 1775766085] req@ffff8b2382dadf80 x1862022098887424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766085.688068:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098887424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766085.688077:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766085.688081:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766085.688085:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098887424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766085.688091:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098887552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766085.688107:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766085.688115:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766085/real 1775766085] req@ffff8b2382daf480 x1862022098887552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766085.688123:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098887552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766085.688128:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766085.688130:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766085.688132:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098887552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766085.688156:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098887680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766085.688167:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766085.688173:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766085/real 1775766085] req@ffff8b2382dace00 x1862022098887680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766085.688180:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098887680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766085.688186:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766085.688187:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766085.688189:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098887680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766085.688193:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098887808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766085.688205:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766085.688210:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766085/real 1775766085] req@ffff8b2382dafb80 x1862022098887808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766085.688218:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098887808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766085.688223:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766085.688224:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766085.688226:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098887808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766085.688230:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098887936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766085.688240:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766085.688244:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766085/real 1775766085] req@ffff8b2382dac700 x1862022098887936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766085.688251:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098887936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766085.688256:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766085.688258:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766085.688260:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098887936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766086.583754:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766087.607660:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766088.631748:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766089.655630:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766090.679640:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766090.807560:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766090.807569:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766090.807574:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766090.807583:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3552 00000100:00080000:3.0:1775766090.807593:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766090.807600:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766090.807637:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098888064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766090.807667:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766090.807670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766090.807672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766090.807675:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3552 00000100:00080000:3.0:1775766090.807678:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766090.807682:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766090.807690:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766090.807700:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098888192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766090.807711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766090.807716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775766090.807719:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098888064:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766090.807719:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766090.807724:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3552 00000100:00080000:3.0:1775766090.807729:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766090.807734:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766090.807757:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766090.807786:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098888320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766090.807793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766090.807795:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766090.807797:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766090.807801:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3552 00000100:00080000:3.0:1775766090.807804:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766090.807807:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766090.807811:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766090.807821:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098888448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766090.807826:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766090.807837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766090.807841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766090.807843:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766090.807849:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3552 00000100:00080000:3.0:1775766090.807853:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766090.807858:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766090.807864:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766090.807866:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766090/real 1775766090] req@ffff8b23864f1c00 x1862022098888064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766090.807880:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098888576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766090.807883:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098888064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766090.807906:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766090.807910:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766090.807913:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098888064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766090.807932:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098888192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766090.807948:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766090.807956:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766090/real 1775766090] req@ffff8b23864f0380 x1862022098888192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766090.807964:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098888192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766090.807969:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766090.807972:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766090.807974:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098888192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766090.807977:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098888320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766090.807987:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766090.807992:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766090/real 1775766090] req@ffff8b23864f1f80 x1862022098888320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766090.807999:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098888320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766090.808004:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766090.808006:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766090.808008:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098888320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766090.808011:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098888448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766090.808021:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766090.808026:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766090/real 1775766090] req@ffff8b23864f2680 x1862022098888448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766090.808038:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098888448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766090.808042:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766090.808044:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766090.808046:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098888448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766090.808049:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098888576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766090.808058:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766090.808063:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766090/real 1775766090] req@ffff8b23864f0a80 x1862022098888576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766090.808070:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098888576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766090.808075:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766090.808076:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766090.808078:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098888576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766091.703703:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766092.727614:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766093.751667:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766094.775779:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766095.799735:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766095.927603:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766095.927625:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766095.927630:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766095.927641:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3557 00000100:00080000:1.0:1775766095.927654:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766095.927661:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766095.927742:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098888704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766095.927778:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766095.927783:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766095.927786:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766095.927791:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3557 00000100:00080000:1.0:1775766095.927796:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766095.927800:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766095.927812:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766095.927833:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098888832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766095.927848:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766095.927853:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766095.927856:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766095.927862:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3557 00000100:00080000:1.0:1775766095.927867:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766095.927875:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098888704:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766095.927877:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766095.927890:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766095.927910:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098888960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766095.927948:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766095.927953:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766095.927955:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766095.927959:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3557 00000100:00080000:1.0:1775766095.927975:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766095.927979:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766095.927986:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766095.927988:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766095.928006:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098889088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766095.928043:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766095.928047:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766095.928050:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766095.928056:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3557 00000100:00080000:1.0:1775766095.928063:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766095.928068:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766095.928076:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766095/real 1775766095] req@ffff8b23864fce00 x1862022098888704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775766095.928077:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766095.928102:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098889216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766095.928115:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098888704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766095.928157:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766095.928164:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766095.928170:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098888704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766095.928197:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098888832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766095.928224:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766095.928238:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766095/real 1775766095] req@ffff8b23864fe680 x1862022098888832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766095.928251:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098888832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766095.928260:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766095.928263:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766095.928267:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098888832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766095.928292:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098888960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766095.928309:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766095.928318:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766095/real 1775766095] req@ffff8b23864fca80 x1862022098888960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766095.928330:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098888960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766095.928338:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766095.928341:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766095.928344:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098888960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766095.928352:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098889088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766095.928368:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766095.928376:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766095/real 1775766095] req@ffff8b23864fea00 x1862022098889088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766095.928388:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098889088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766095.928396:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766095.928420:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766095.928424:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098889088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766095.928431:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098889216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766095.928451:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766095.928461:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766095/real 1775766095] req@ffff8b23864fd180 x1862022098889216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766095.928474:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098889216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766095.928484:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766095.928552:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766095.928557:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098889216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766096.823773:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766097.847754:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766098.871719:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766099.895914:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766100.919745:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766101.047673:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766101.047690:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766101.047698:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766101.047714:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3562 00000100:00080000:3.0:1775766101.047731:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766101.047739:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766101.047789:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098889344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766101.047830:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766101.047835:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766101.047837:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766101.047842:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3562 00000100:00080000:3.0:1775766101.047847:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766101.047851:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766101.047863:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766101.047880:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098889472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766101.047897:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766101.047902:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766101.047904:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766101.047914:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098889344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766101.047919:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3562 00000100:00080000:3.0:1775766101.047933:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766101.047941:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766101.047951:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766101.047977:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098889600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766101.047998:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766101.048006:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766101.048010:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766101.048018:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3562 00000800:00000100:0.0:1775766101.048026:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766101.048034:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766101.048046:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766101.048057:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766101.048084:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098889728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766101.048104:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766101.048112:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766101/real 1775766101] req@ffff8b23864f1180 x1862022098889344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766101.048113:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766101.048121:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766101.048139:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3562 00000100:00080000:3.0:1775766101.048153:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766101.048163:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766101.048171:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766101.048177:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098889344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766101.048195:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098889856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766101.048214:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766101.048226:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766101.048233:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098889344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766101.048269:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098889472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766101.048308:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766101.048326:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766101/real 1775766101] req@ffff8b23864f0700 x1862022098889472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766101.048342:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098889472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766101.048353:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766101.048357:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766101.048362:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098889472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766101.048369:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098889600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766101.048389:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766101.048401:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766101/real 1775766101] req@ffff8b23864f0000 x1862022098889600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766101.048415:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098889600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766101.048433:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766101.048437:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766101.048441:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098889600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766101.048447:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098889728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766101.048523:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766101.048537:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766101/real 1775766101] req@ffff8b23864f3b80 x1862022098889728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766101.048552:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098889728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766101.048562:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766101.048566:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766101.048570:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098889728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766101.048579:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098889856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766101.048614:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766101.048675:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766101/real 1775766101] req@ffff8b23864f3100 x1862022098889856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766101.048702:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098889856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766101.048719:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766101.048726:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766101.048733:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098889856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766101.943705:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766102.967820:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766103.991772:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766105.015667:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766106.039649:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766106.167514:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766106.167522:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766106.167526:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766106.167534:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3567 00000100:00080000:3.0:1775766106.167544:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766106.167549:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766106.167583:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098889984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766106.167631:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766106.167634:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766106.167635:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766106.167638:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3567 00000100:00080000:3.0:1775766106.167640:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766106.167643:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766106.167651:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766106.167662:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098890112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766106.167671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766106.167674:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766106.167676:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766106.167677:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098889984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766106.167681:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3567 00000100:00080000:3.0:1775766106.167687:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766106.167689:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766106.167692:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766106.167702:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098890240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766106.167708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766106.167711:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766106.167713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766106.167716:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3567 00000100:00080000:3.0:1775766106.167719:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766106.167721:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766106.167724:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766106.167731:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098890368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766106.167737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775766106.167741:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766106.167742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766106.167745:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766106.167748:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3567 00000100:00080000:3.0:1775766106.167752:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766106.167754:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766106.167758:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766106.167766:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098890496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766106.167788:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766106/real 1775766106] req@ffff8b23864f2a00 x1862022098889984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766106.167809:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098889984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766106.167821:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766106.167825:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766106.167829:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098889984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766106.167849:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098890112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766106.167866:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766106.167875:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766106/real 1775766106] req@ffff8b23864f2d80 x1862022098890112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766106.167883:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098890112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766106.167889:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766106.167891:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766106.167893:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098890112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766106.167897:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098890240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766106.167907:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766106.167913:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766106/real 1775766106] req@ffff8b23864f3480 x1862022098890240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766106.167920:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098890240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766106.167925:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766106.167926:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766106.167928:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098890240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766106.167932:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098890368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766106.167942:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766106.167947:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766106/real 1775766106] req@ffff8b23864f3100 x1862022098890368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766106.167954:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098890368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766106.167959:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766106.167961:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766106.167962:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098890368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766106.167966:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098890496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766106.167975:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766106.167979:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766106/real 1775766106] req@ffff8b23864f3b80 x1862022098890496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766106.167985:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098890496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766106.167991:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766106.167993:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766106.167995:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098890496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766107.063647:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766108.087713:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766109.111719:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766110.135639:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766111.159643:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766111.287647:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766111.287661:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766111.287666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766111.287679:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3572 00000100:00080000:0.0:1775766111.287694:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766111.287701:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766111.287778:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098890624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766111.287845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766111.287850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766111.287852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766111.287856:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3572 00000100:00080000:0.0:1775766111.287861:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766111.287865:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766111.287878:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766111.287894:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098890752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766111.287918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766111.287922:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766111.287924:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766111.287931:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098890624:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766111.287934:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3572 00000100:00080000:0.0:1775766111.287941:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766111.287946:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766111.287950:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766111.287961:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098890880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766111.287969:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766111.287973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766111.287974:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766111.287978:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3572 00000100:00080000:0.0:1775766111.287981:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766111.287984:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766111.287989:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766111.287997:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098891008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766111.288004:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766111.288008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766111.288010:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766111.288013:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3572 00000100:00080000:0.0:1775766111.288017:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766111.288019:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766111.288023:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766111.288028:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766111.288040:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098891136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766111.288085:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766111/real 1775766111] req@ffff8b2382dac700 x1862022098890624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766111.288112:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098890624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766111.288123:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766111.288128:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766111.288132:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098890624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766111.288159:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098890752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766111.288180:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766111.288203:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766111/real 1775766111] req@ffff8b2382dafb80 x1862022098890752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766111.288214:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098890752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766111.288222:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766111.288224:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766111.288227:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098890752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766111.288232:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098890880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766111.288249:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766111.288257:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766111/real 1775766111] req@ffff8b2382dace00 x1862022098890880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766111.288267:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098890880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766111.288280:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766111.288284:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766111.288286:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098890880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766111.288291:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098891008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766111.288305:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766111.288312:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766111/real 1775766111] req@ffff8b2382daf480 x1862022098891008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766111.288321:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098891008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766111.288328:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766111.288330:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766111.288333:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098891008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766111.288337:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098891136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766111.288349:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766111.288356:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766111/real 1775766111] req@ffff8b2382dadf80 x1862022098891136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766111.288364:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098891136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766111.288371:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766111.288373:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766111.288377:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098891136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766112.183627:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766113.207660:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766114.231750:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766115.255675:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766116.279782:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766116.407687:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766116.407700:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766116.407706:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766116.407718:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3577 00000100:00080000:1.0:1775766116.407731:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766116.407740:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766116.407809:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098891264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766116.407845:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766116.407850:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766116.407853:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766116.407857:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3577 00000100:00080000:1.0:1775766116.407878:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766116.407882:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766116.407893:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766116.407909:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098891392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766116.407925:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766116.407929:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766116.407931:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766116.407941:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3577 00000100:00080000:1.0:1775766116.407944:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766116.407957:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766116.407961:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098891264:192.168.202.128@tcp:250:kworker.0 00000008:00100000:1.0:1775766116.407963:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766116.407982:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098891520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766116.407993:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766116.407997:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766116.407999:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766116.408004:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3577 00000100:00080000:1.0:1775766116.408009:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766116.408012:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766116.408018:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766116.408029:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098891648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766116.408039:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766116.408043:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766116.408045:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766116.408049:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3577 00000100:00080000:1.0:1775766116.408054:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766116.408058:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766116.408066:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766116.408104:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098891776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766116.408128:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098891392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766116.408188:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098891520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766116.408251:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098891648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766116.408308:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098891776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766117.303679:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766118.327586:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766119.351834:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775766119.479903:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775766119.479962:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766119.479984:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766119.480033:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766119.480042:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766119.480050:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766119.480062:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:3.0:1775766119.480105:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766116/real 1775766119] req@ffff8b23864ff800 x1862022098891776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766119.480148:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098891776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766119.480188:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766119.480197:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766119.480204:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098891776:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766119.480215:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766116/real 1775766119] req@ffff8b23864fc380 x1862022098891392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766119.480236:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098891392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766119.480255:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766119.480260:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766119.480265:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098891392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766119.480276:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766116/real 1775766119] req@ffff8b23864fed80 x1862022098891520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766119.480292:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098891520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766119.480302:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766119.480307:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766119.480311:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098891520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766119.480319:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766116/real 1775766119] req@ffff8b23864fc000 x1862022098891648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766119.480334:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098891648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766119.480344:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766119.480348:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766119.480351:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098891648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766119.480360:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766116/real 1775766119] req@ffff8b23864fdf80 x1862022098891264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766119.480375:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098891264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766119.480384:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766119.480389:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766119.480393:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098891264:192.168.202.128@tcp:250:kworker.0 00010000:00010000:3.0:1775766120.375673:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766121.399821:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766121.527609:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766121.527624:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766121.527630:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766121.527646:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3582 00000100:00080000:1.0:1775766121.527666:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766121.527679:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766121.527763:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098891904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766121.527836:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766121.527843:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766121.527847:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766121.527857:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3582 00000100:00080000:1.0:1775766121.527866:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766121.527874:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766121.527892:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766121.527906:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098891904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775766121.527928:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098892032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766121.527954:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766121.527962:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766121.527967:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766121.527976:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3582 00000100:00080000:1.0:1775766121.527988:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766121.527996:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775766121.528005:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:1.0:1775766121.528010:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766121.528044:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098892160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766121.528066:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766121.528073:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766121.528078:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766121.528087:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766121/real 1775766121] req@ffff8b23864fdf80 x1862022098891904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766121.528092:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3582 00000100:00080000:1.0:1775766121.528106:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766121.528121:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766121.528130:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766121.528168:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098892288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766121.528175:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098891904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766121.528208:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766121.528219:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766121.528223:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766121.528224:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766121.528240:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3582 00000100:00080000:1.0:1775766121.528249:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766121.528258:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00080000:1.0:1775766121.528265:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766121.528275:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098891904:192.168.202.128@tcp:250:kworker.0 00000008:00100000:1.0:1775766121.528285:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766121.528314:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098892416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766121.528323:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098892032:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766121.528403:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766121.528429:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766121/real 1775766121] req@ffff8b23864fc000 x1862022098892032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766121.528557:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098892032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766121.528572:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766121.528579:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766121.528585:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098892032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766121.528597:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098892160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766121.528637:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766121.528656:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766121/real 1775766121] req@ffff8b23864fed80 x1862022098892160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766121.528675:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098892160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766121.528688:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766121.528693:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766121.528699:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098892160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766121.528709:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098892288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766121.528740:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766121.528758:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766121/real 1775766121] req@ffff8b23864fc380 x1862022098892288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766121.528780:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098892288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766121.528793:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766121.528798:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766121.528803:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098892288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766121.529420:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098892416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766121.529521:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766121.529539:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766121/real 1775766121] req@ffff8b23864ff800 x1862022098892416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766121.529554:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098892416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766121.529566:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766121.529572:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766121.529578:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098892416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766122.423684:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766123.447629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766124.471617:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766125.495746:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766126.519630:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766126.647596:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766126.647607:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766126.647611:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766126.647622:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3588 00000100:00080000:3.0:1775766126.647633:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766126.647640:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766126.647681:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098892544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766126.647717:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766126.647721:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766126.647723:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766126.647728:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3588 00000100:00080000:3.0:1775766126.647732:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766126.647736:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766126.647747:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766126.647761:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098892672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766126.647770:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098892544:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766126.647779:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766126.647783:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766126.647785:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766126.647791:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3588 00000100:00080000:3.0:1775766126.647795:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766126.647800:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766126.647806:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766126.647819:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098892800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766126.647829:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766126.647829:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766126.647836:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766126.647839:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766126.647844:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3588 00000100:00080000:3.0:1775766126.647850:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766126.647854:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766126.647860:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766126.647875:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098892928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766126.647881:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766126/real 1775766126] req@ffff8b23864f0000 x1862022098892544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766126.647892:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766126.647900:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766126.647905:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766126.647910:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3588 00000100:00080000:3.0:1775766126.647915:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766126.647916:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098892544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766126.647922:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766126.647932:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775766126.647933:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766126.647941:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766126.647945:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098892544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766126.647950:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098893056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766126.647970:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098892672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766126.647989:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766126.647999:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766126/real 1775766126] req@ffff8b23864f0700 x1862022098892672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766126.648007:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098892672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766126.648012:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766126.648014:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766126.648016:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098892672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766126.648020:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098892800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766126.648034:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766126.648040:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766126/real 1775766126] req@ffff8b23864f1180 x1862022098892800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766126.648047:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098892800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766126.648052:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766126.648053:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766126.648055:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098892800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766126.648058:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098892928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766126.648069:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766126.648075:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766126/real 1775766126] req@ffff8b23864f0a80 x1862022098892928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766126.648082:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098892928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766126.648087:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766126.648088:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766126.648090:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098892928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766126.648093:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098893056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766126.648102:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766126.648107:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766126/real 1775766126] req@ffff8b23864f2680 x1862022098893056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766126.648119:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098893056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766126.648124:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766126.648126:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766126.648127:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098893056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766127.543639:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766128.567684:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766129.591612:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766130.615657:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766131.639579:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766131.767557:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766131.767565:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766131.767568:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766131.767576:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3593 00000100:00080000:1.0:1775766131.767585:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766131.767590:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766131.767625:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098893184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766131.767647:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766131.767650:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766131.767652:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766131.767656:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3593 00000100:00080000:1.0:1775766131.767659:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766131.767663:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766131.767673:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766131.767685:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098893312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766131.767691:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098893184:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766131.767696:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766131.767699:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766131.767701:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766131.767705:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3593 00000100:00080000:1.0:1775766131.767708:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766131.767712:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766131.767716:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766131.767729:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098893440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766131.767738:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766131.767741:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766131.767742:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775766131.767746:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775766131.767747:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3593 00000100:00080000:1.0:1775766131.767749:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766131.767752:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766131.767756:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766131.767765:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098893568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766131.767773:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766131.767776:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766131.767778:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766131.767781:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3593 00000100:00100000:0.0:1775766131.767783:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766131/real 1775766131] req@ffff8b23864fd180 x1862022098893184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766131.767784:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766131.767788:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766131.767792:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766131.767800:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098893184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766131.767810:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1775766131.767812:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098893696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766131.767814:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766131.767819:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098893184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766131.767841:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098893312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766131.767865:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766131.767876:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766131/real 1775766131] req@ffff8b23864fea00 x1862022098893312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766131.767884:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098893312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766131.767893:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766131.767896:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766131.767898:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098893312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766131.767902:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098893440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766131.767919:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766131.767930:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766131/real 1775766131] req@ffff8b23864fca80 x1862022098893440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766131.767942:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098893440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766131.767951:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766131.767955:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766131.767959:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098893440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766131.767965:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098893568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766131.767984:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766131.767992:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766131/real 1775766131] req@ffff8b23864fe680 x1862022098893568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766131.767999:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098893568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766131.768005:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766131.768007:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766131.768009:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098893568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766131.768013:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098893696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766131.768024:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766131.768029:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766131/real 1775766131] req@ffff8b23864fce00 x1862022098893696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766131.768036:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098893696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766131.768041:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766131.768043:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766131.768044:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098893696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766132.663668:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766133.687593:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766134.711675:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766135.735618:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766136.759636:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766136.887536:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766136.887544:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766136.887548:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766136.887556:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3598 00000100:00080000:1.0:1775766136.887566:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766136.887571:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766136.887603:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098893824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766136.887633:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766136.887637:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766136.887639:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766136.887643:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3598 00000100:00080000:1.0:1775766136.887647:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766136.887651:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766136.887660:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766136.887674:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098893952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766136.887683:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098893824:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766136.887686:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766136.887689:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766136.887691:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766136.887695:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3598 00000100:00080000:1.0:1775766136.887698:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766136.887702:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766136.887706:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766136.887716:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098894080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766136.887724:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766136.887727:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766136.887729:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766136.887732:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3598 00000100:00080000:1.0:1775766136.887735:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775766136.887739:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775766136.887739:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766136.887752:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766136.887765:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098894208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766136.887774:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766136.887777:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1775766136.887778:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766136/real 1775766136] req@ffff8b23864fdc00 x1862022098893824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766136.887779:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766136.887783:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3598 00000100:00080000:1.0:1775766136.887786:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766136.887790:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766136.887794:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766136.887797:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098893824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766136.887803:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098894336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766136.887807:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766136.887812:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766136.887825:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098893824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766136.887853:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098893952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766136.887871:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766136.887881:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766136/real 1775766136] req@ffff8b23864fd500 x1862022098893952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766136.887890:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098893952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766136.887895:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766136.887897:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766136.887900:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098893952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766136.887904:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098894080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766136.887914:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766136.887920:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766136/real 1775766136] req@ffff8b23864ff100 x1862022098894080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766136.887927:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098894080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766136.887932:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766136.887934:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766136.887936:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098894080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766136.887939:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098894208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766136.887950:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766136.887955:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766136/real 1775766136] req@ffff8b23864fe300 x1862022098894208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766136.887962:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098894208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766136.887967:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766136.887969:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766136.887972:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098894208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766136.887975:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098894336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766136.887984:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766136.887989:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766136/real 1775766136] req@ffff8b23864ffb80 x1862022098894336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766136.887996:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098894336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766136.888001:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766136.888003:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766136.888005:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098894336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766137.783659:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766138.807684:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766139.831672:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766140.855642:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766141.879616:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766142.007622:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766142.007633:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766142.007638:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766142.007649:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3603 00000100:00080000:3.0:1775766142.007663:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766142.007669:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766142.007714:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098894464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766142.007744:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766142.007747:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766142.007749:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766142.007752:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3603 00000100:00080000:3.0:1775766142.007755:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766142.007758:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766142.007767:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766142.007778:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098894592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766142.007786:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766142.007789:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766142.007790:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766142.007794:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3603 00000100:00080000:3.0:1775766142.007796:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766142.007798:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766142.007802:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766142.007809:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098894720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766142.007816:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766142.007819:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766142.007820:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766142.007823:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3603 00000100:00080000:3.0:1775766142.007825:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766142.007829:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766142.007830:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098894464:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775766142.007833:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766142.007844:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098894848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766142.007855:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766142.007858:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766142.007859:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766142.007861:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3603 00000100:00080000:3.0:1775766142.007863:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766142.007865:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766142.007869:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766142.007877:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098894976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766142.007911:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766142.007955:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766142/real 1775766142] req@ffff8b23864f1f80 x1862022098894464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766142.007971:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098894464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766142.007979:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766142.007983:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766142.007987:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098894464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766142.007993:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098894592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766142.008008:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766142.008017:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766142/real 1775766142] req@ffff8b23864f0380 x1862022098894592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766142.008024:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098894592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766142.008030:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766142.008032:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766142.008035:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098894592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766142.008038:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098894720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766142.008048:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766142.008054:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766142/real 1775766142] req@ffff8b23864f1c00 x1862022098894720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766142.008061:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098894720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766142.008066:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766142.008068:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766142.008070:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098894720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766142.008094:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098894848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766142.008105:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766142.008111:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766142/real 1775766142] req@ffff8b23864f2680 x1862022098894848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766142.008119:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098894848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766142.008124:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766142.008126:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766142.008128:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098894848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766142.008131:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098894976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766142.008141:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766142.008147:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766142/real 1775766142] req@ffff8b23864f0a80 x1862022098894976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766142.008154:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098894976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766142.008159:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766142.008160:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766142.008162:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098894976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766142.903608:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766143.927713:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766144.951610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766145.975656:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766146.999614:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766147.127631:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766147.127643:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766147.127648:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766147.127659:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3608 00000100:00080000:1.0:1775766147.127673:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766147.127681:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766147.127728:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098895104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766147.127756:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766147.127760:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766147.127762:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766147.127765:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3608 00000100:00080000:1.0:1775766147.127768:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766147.127770:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766147.127780:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766147.127793:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098895232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766147.127803:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766147.127807:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766147.127809:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766147.127813:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3608 00000100:00080000:1.0:1775766147.127816:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766147.127821:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766147.127827:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766147.127837:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098895104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775766147.127838:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098895360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766147.127864:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766147.127867:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766147.127869:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766147.127873:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3608 00000100:00080000:1.0:1775766147.127874:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766147.127877:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766147.127882:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766147.127891:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098895488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766147.127899:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766147.127903:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766147.127905:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766147.127910:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3608 00000100:00080000:1.0:1775766147.127923:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766147.127942:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766147.127947:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766147.127948:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766147.127964:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098895616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766147.128012:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766147/real 1775766147] req@ffff8b23864ffb80 x1862022098895104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766147.128029:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098895104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766147.128042:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766147.128045:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766147.128048:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098895104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766147.128054:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098895232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766147.128073:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766147.128082:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766147/real 1775766147] req@ffff8b23864fe300 x1862022098895232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766147.128090:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098895232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766147.128106:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766147.128109:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766147.128111:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098895232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766147.128131:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098895360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766147.128143:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766147.128149:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766147/real 1775766147] req@ffff8b23864ff100 x1862022098895360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766147.128166:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098895360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766147.128172:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766147.128175:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766147.128177:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098895360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766147.128181:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098895488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766147.128194:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766147.128215:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766147/real 1775766147] req@ffff8b23864fd500 x1862022098895488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766147.128225:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098895488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766147.128231:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766147.128233:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766147.128235:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098895488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766147.128240:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098895616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766147.128256:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766147.128262:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766147/real 1775766147] req@ffff8b23864fdc00 x1862022098895616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766147.128270:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098895616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766147.128275:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766147.128277:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766147.128279:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098895616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766148.023767:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766149.047639:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766150.071684:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766151.095644:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766152.119719:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766152.247664:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766152.247676:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766152.247681:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766152.247692:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3613 00000100:00080000:1.0:1775766152.247707:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766152.247714:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766152.247762:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098895744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766152.247810:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766152.247813:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766152.247814:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766152.247817:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3613 00000100:00080000:1.0:1775766152.247820:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766152.247823:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766152.247833:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766152.247842:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098895872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766152.247851:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766152.247855:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766152.247857:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766152.247861:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3613 00000100:00080000:1.0:1775766152.247864:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766152.247870:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766152.247874:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098895744:192.168.202.128@tcp:250:kworker.0 00000008:00100000:1.0:1775766152.247874:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766152.247892:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098896000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766152.247901:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766152.247903:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766152.247905:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766152.247909:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3613 00000100:00080000:1.0:1775766152.247912:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766152.247915:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766152.247918:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766152.247933:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098896128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766152.247938:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766152.247941:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766152.247942:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766152.247945:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3613 00000100:00080000:1.0:1775766152.247948:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766152.247951:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766152.247954:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766152.247966:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098896256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766152.247989:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766152.248034:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766152/real 1775766152] req@ffff8b23864fce00 x1862022098895744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766152.248049:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098895744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766152.248065:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766152.248069:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766152.248072:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098895744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766152.248079:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098895872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766152.248098:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766152.248108:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766152/real 1775766152] req@ffff8b23864fe680 x1862022098895872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766152.248121:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098895872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766152.248129:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766152.248132:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766152.248143:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098895872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766152.248187:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098896000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766152.248213:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766152.248222:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766152/real 1775766152] req@ffff8b23864fca80 x1862022098896000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766152.248230:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098896000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766152.248241:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766152.248243:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766152.248245:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098896000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766152.248250:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098896128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766152.248262:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766152.248267:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766152/real 1775766152] req@ffff8b23864fea00 x1862022098896128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766152.248275:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098896128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766152.248280:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766152.248282:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766152.248283:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098896128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766152.248287:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098896256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766152.248298:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766152.248303:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766152/real 1775766152] req@ffff8b23864fd180 x1862022098896256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766152.248314:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098896256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766152.248322:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766152.248325:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766152.248329:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098896256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766153.143677:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766154.167741:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766155.191607:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766156.215664:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766157.239593:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766157.367622:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766157.367631:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766157.367635:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766157.367643:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3618 00000100:00080000:3.0:1775766157.367653:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766157.367657:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766157.367690:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098896384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766157.367721:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766157.367724:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766157.367725:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766157.367728:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3618 00000100:00080000:3.0:1775766157.367730:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766157.367733:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766157.367740:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766157.367749:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098896512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766157.367758:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766157.367761:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766157.367762:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766157.367765:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3618 00000100:00080000:3.0:1775766157.367767:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766157.367772:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098896384:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766157.367772:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766157.367775:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766157.367784:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098896640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766157.367791:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766157.367794:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766157.367795:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766157.367797:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3618 00000100:00080000:3.0:1775766157.367800:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766157.367803:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766157.367806:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766157.367813:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098896768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766157.367819:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766157.367821:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766157.367822:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766157.367824:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3618 00000100:00080000:3.0:1775766157.367828:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766157.367830:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766157.367832:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766157.367838:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766157.367841:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098896896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766157.367878:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766157/real 1775766157] req@ffff8b23864f1180 x1862022098896384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766157.367898:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098896384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766157.367906:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766157.367909:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766157.367912:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098896384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766157.367917:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098896512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766157.367936:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766157.367948:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766157/real 1775766157] req@ffff8b23864f0700 x1862022098896512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766157.367962:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098896512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766157.367971:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766157.367974:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766157.367978:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098896512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766157.368003:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098896640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766157.368022:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766157.368031:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766157/real 1775766157] req@ffff8b23864f0000 x1862022098896640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766157.368044:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098896640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766157.368052:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766157.368056:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766157.368059:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098896640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766157.368065:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098896768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766157.368087:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766157.368123:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766157/real 1775766157] req@ffff8b23864f3b80 x1862022098896768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766157.368136:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098896768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766157.368144:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766157.368147:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766157.368149:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098896768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766157.368153:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098896896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766157.368170:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766157.368177:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766157/real 1775766157] req@ffff8b23864f3100 x1862022098896896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766157.368185:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098896896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766157.368198:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766157.368201:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766157.368203:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098896896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766158.263650:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766159.287625:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766160.311635:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766161.335608:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766162.359626:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766162.487590:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766162.487605:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766162.487613:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766162.487628:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3623 00000100:00080000:3.0:1775766162.487645:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766162.487656:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766162.487715:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098897024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766162.487768:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766162.487776:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766162.487780:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766162.487787:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3623 00000100:00080000:3.0:1775766162.487795:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766162.487802:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766162.487820:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766162.487845:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098897152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766162.487880:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098897024:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766162.487882:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766162.487888:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766162.487892:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766162.487898:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3623 00000100:00080000:3.0:1775766162.487904:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766162.487911:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766162.487918:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766162.487935:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098897280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766162.487949:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766162.487955:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766162.487958:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766162.487966:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3623 00000100:00080000:3.0:1775766162.487972:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766162.487977:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766162.487985:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766162.488003:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766162.488003:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098897408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766162.488017:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766162.488022:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766162.488025:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766162.488031:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3623 00000100:00080000:3.0:1775766162.488036:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766162.488042:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766162.488049:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766162.488101:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766162/real 1775766162] req@ffff8b23864f3480 x1862022098897024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766162.488101:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098897536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766162.488135:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098897024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766162.488164:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766162.488172:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766162.488179:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098897024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766162.488218:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098897152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766162.488275:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766162.488292:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766162/real 1775766162] req@ffff8b23864f2d80 x1862022098897152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766162.488307:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098897152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766162.488317:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766162.488321:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766162.488325:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098897152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766162.488333:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098897280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766162.488352:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766162.488362:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766162/real 1775766162] req@ffff8b23864f2a00 x1862022098897280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766162.488376:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098897280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766162.488384:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766162.488387:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766162.488391:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098897280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766162.488407:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098897408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766162.488427:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766162.488436:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766162/real 1775766162] req@ffff8b23864f3100 x1862022098897408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766162.488449:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098897408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766162.488502:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766162.488507:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766162.488511:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098897408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766162.488518:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098897536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766162.488549:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766162.488561:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766162/real 1775766162] req@ffff8b23864f3b80 x1862022098897536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766162.488575:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098897536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766162.488585:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766162.488588:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766162.488591:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098897536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766163.383574:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766164.407609:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766165.431629:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766166.455611:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766167.479705:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766167.607654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766167.607663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766167.607666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766167.607676:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3629 00000100:00080000:0.0:1775766167.607686:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766167.607692:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766167.607728:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098897664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766167.607758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766167.607761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766167.607763:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766167.607765:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3629 00000100:00080000:0.0:1775766167.607768:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766167.607770:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766167.607778:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766167.607793:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098897792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766167.607811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766167.607814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766167.607815:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766167.607818:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3629 00000100:00080000:0.0:1775766167.607821:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766167.607823:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766167.607827:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766167.607838:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098897920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766167.607847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766167.607852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766167.607854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766167.607863:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3629 00000100:00100000:3.0:1775766167.607867:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098897664:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766167.607868:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766167.607872:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766167.607882:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766167.607904:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098898048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766167.607910:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766167.607913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766167.607914:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766167.607918:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3629 00000100:00080000:0.0:1775766167.607920:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766167.607922:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766167.607926:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766167.607937:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098898176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766167.608063:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766167.608141:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766167/real 1775766167] req@ffff8b2382daea00 x1862022098897664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766167.608178:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098897664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766167.608211:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766167.608220:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766167.608228:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098897664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766167.608244:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098897792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766167.608292:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766167.608317:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766167/real 1775766167] req@ffff8b2382dad500 x1862022098897792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766167.608343:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098897792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766167.608359:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766167.608366:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766167.608373:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098897792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766167.608423:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098897920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766167.608504:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766167.608535:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766167/real 1775766167] req@ffff8b2382dae680 x1862022098897920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766167.608558:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098897920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766167.608573:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766167.608580:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766167.608587:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098897920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766167.608600:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098898048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766167.608642:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766167.608663:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766167/real 1775766167] req@ffff8b2382dac380 x1862022098898048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766167.608686:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098898048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766167.608700:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766167.608707:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766167.608713:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098898048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766167.608725:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098898176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766167.608760:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766167.608774:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766167/real 1775766167] req@ffff8b2382daed80 x1862022098898176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766167.608788:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098898176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766167.608798:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766167.608801:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766167.608806:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098898176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766168.503657:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766169.527584:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766170.551703:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766171.575666:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766172.599636:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766172.727592:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766172.727601:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766172.727605:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766172.727613:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3634 00000100:00080000:1.0:1775766172.727623:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766172.727629:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766172.727666:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098898304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766172.727687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766172.727690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766172.727692:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766172.727694:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3634 00000100:00080000:1.0:1775766172.727697:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766172.727699:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766172.727708:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766172.727717:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098898432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766172.727726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766172.727729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766172.727730:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766172.727734:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3634 00000100:00080000:1.0:1775766172.727736:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766172.727740:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766172.727743:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098898304:192.168.202.128@tcp:250:kworker.0 00000008:00100000:1.0:1775766172.727744:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766172.727754:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098898560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766172.727762:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766172.727765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766172.727766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766172.727769:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3634 00000100:00080000:1.0:1775766172.727771:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766172.727773:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766172.727776:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766172.727785:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098898688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766172.727791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766172.727792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766172.727794:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766172.727797:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3634 00000100:00080000:1.0:1775766172.727807:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766172.727809:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766172.727814:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766172.727814:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766172.727825:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098898816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766172.727866:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766172/real 1775766172] req@ffff8b23864ff800 x1862022098898304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766172.727888:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098898304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766172.727900:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766172.727904:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766172.727908:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098898304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766172.727917:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098898432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766172.727941:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766172.727953:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766172/real 1775766172] req@ffff8b23864fc380 x1862022098898432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766172.727986:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098898432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766172.727994:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766172.727998:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766172.728002:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098898432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766172.728028:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098898560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766172.728047:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766172.728058:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766172/real 1775766172] req@ffff8b23864fed80 x1862022098898560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766172.728078:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098898560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766172.728086:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766172.728090:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766172.728093:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098898560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766172.728099:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098898688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766172.728118:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766172.728128:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766172/real 1775766172] req@ffff8b23864fc000 x1862022098898688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766172.728140:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098898688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766172.728149:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766172.728152:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766172.728155:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098898688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766172.728161:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098898816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766172.728182:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766172.728192:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766172/real 1775766172] req@ffff8b23864fdf80 x1862022098898816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766172.728205:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098898816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766172.728220:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766172.728223:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766172.728226:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098898816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766173.623659:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766174.647698:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766175.671723:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766176.695633:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766177.719742:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766177.847559:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766177.847570:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766177.847574:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766177.847584:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3639 00000100:00080000:1.0:1775766177.847596:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766177.847603:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766177.847661:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098898944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766177.847711:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766177.847715:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766177.847717:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766177.847726:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3639 00000100:00080000:1.0:1775766177.847730:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766177.847734:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766177.847743:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766177.847777:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098899072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766177.847792:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766177.847800:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1775766177.847800:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098898944:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766177.847806:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766177.847817:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3639 00000100:00080000:1.0:1775766177.847821:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766177.847824:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766177.847830:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766177.847849:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098899200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766177.847885:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766177.847888:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766177.847889:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766177.847893:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3639 00000100:00080000:1.0:1775766177.847897:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766177.847900:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766177.847905:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766177.847935:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766177.847942:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098899328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766177.847953:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766177.847956:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766177.847958:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766177.847962:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3639 00000100:00080000:1.0:1775766177.847966:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766177.847970:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766177.847976:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766177.847983:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766177/real 1775766177] req@ffff8b23864fdf80 x1862022098898944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766177.847990:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098899456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766177.848022:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098898944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766177.848033:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766177.848038:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766177.848042:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098898944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766177.848065:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098899072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766177.848097:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766177.848108:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766177/real 1775766177] req@ffff8b23864fc000 x1862022098899072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766177.848118:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098899072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766177.848131:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766177.848133:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766177.848136:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098899072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766177.848147:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098899200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766177.848161:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766177.848169:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766177/real 1775766177] req@ffff8b23864fed80 x1862022098899200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766177.848178:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098899200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766177.848184:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766177.848187:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766177.848189:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098899200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766177.848194:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098899328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766177.848207:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766177.848214:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766177/real 1775766177] req@ffff8b23864fc380 x1862022098899328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766177.848223:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098899328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766177.848229:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766177.848231:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766177.848242:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098899328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766177.848247:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098899456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766177.848259:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766177.848265:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766177/real 1775766177] req@ffff8b23864ff800 x1862022098899456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766177.848273:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098899456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766177.848280:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766177.848282:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766177.848285:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098899456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766178.743687:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766179.767604:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766180.791623:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766181.815637:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766182.839744:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766182.967570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766182.967585:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766182.967592:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766182.967608:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3644 00000100:00080000:1.0:1775766182.967625:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766182.967637:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766182.967697:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098899584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766182.967741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766182.967746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766182.967749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766182.967754:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3644 00000100:00080000:1.0:1775766182.967760:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766182.967766:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766182.967778:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766182.967796:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098899712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766182.967811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766182.967815:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766182.967818:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766182.967825:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3644 00000100:00080000:1.0:1775766182.967829:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766182.967834:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766182.967839:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766182.967885:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098899840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766182.967886:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098899584:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766182.967917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766182.967921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766182.967924:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766182.967930:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3644 00000100:00080000:1.0:1775766182.967936:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766182.967941:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766182.967949:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766182.967968:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098899968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766182.967981:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766182.967986:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766182.967988:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766182.967994:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3644 00000100:00080000:1.0:1775766182.968000:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766182.968006:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766182.968016:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766182.968043:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098900096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766182.968045:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098899712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766182.968109:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098899840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766182.968144:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098899968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766182.968170:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098900096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766183.863642:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766184.887674:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766185.911634:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775766186.039694:0:2363:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:0.0:1775766186.039750:0:2363:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775766186.039767:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:0.0:1775766186.039808:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:0.0:1775766186.039819:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:0.0:1775766186.039843:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:0.0:1775766186.039857:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:3.0:1775766186.039868:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766182/real 1775766186] req@ffff8b23864fca80 x1862022098899840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766186.039905:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098899840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766186.039919:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766186.039924:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766186.039928:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098899840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766186.039936:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766182/real 1775766186] req@ffff8b23864fe680 x1862022098899968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766186.039949:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098899968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766186.039957:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766186.039962:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766186.039965:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098899968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766186.039973:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766182/real 1775766186] req@ffff8b23864fce00 x1862022098900096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766186.039984:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098900096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766186.039991:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766186.039995:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766186.039998:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098900096:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766186.040005:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766182/real 1775766186] req@ffff8b23864fd180 x1862022098899584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766186.040020:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098899584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766186.040027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766186.040030:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766186.040047:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098899584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766186.040055:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766182/real 1775766186] req@ffff8b23864fea00 x1862022098899712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766186.040067:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098899712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766186.040074:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766186.040078:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766186.040081:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098899712:192.168.202.128@tcp:38:kworker.0 00010000:00010000:1.0:1775766186.935627:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766187.959668:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766188.087558:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766188.087565:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766188.087569:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766188.087577:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3649 00000100:00080000:1.0:1775766188.087587:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766188.087593:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766188.087624:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098900224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766188.087662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766188.087671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766188.087674:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766188.087678:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3649 00000100:00080000:1.0:1775766188.087683:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766188.087688:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766188.087699:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766188.087710:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098900224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775766188.087717:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098900352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766188.087734:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766188.087738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766188.087740:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766188.087745:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3649 00000100:00080000:1.0:1775766188.087749:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766188.087754:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766188.087759:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766188.087776:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098900480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766188.087781:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775766188.087792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766188.087796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766188.087798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766188.087804:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3649 00000100:00080000:1.0:1775766188.087825:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766188.087829:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766188.087834:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766188.087847:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098900608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766188.087853:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766188/real 1775766188] req@ffff8b23864fdc00 x1862022098900224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766188.087866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766188.087874:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766188.087877:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766188.087882:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3649 00000100:00080000:1.0:1775766188.087888:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766188.087888:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098900224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766188.087895:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766188.087903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775766188.087908:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766188.087915:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766188.087921:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098900224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775766188.087924:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098900736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766188.087959:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098900352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766188.087988:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766188.088004:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766188/real 1775766188] req@ffff8b23864fd500 x1862022098900352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766188.088017:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098900352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766188.088025:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766188.088029:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766188.088032:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098900352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766188.088039:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098900480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766188.088062:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766188.088073:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766188/real 1775766188] req@ffff8b23864ff100 x1862022098900480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766188.088084:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098900480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766188.088092:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766188.088095:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766188.088098:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098900480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766188.088103:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098900608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766188.088123:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766188.088134:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766188/real 1775766188] req@ffff8b23864fe300 x1862022098900608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766188.088146:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098900608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766188.088154:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766188.088158:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766188.088161:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098900608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766188.088166:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098900736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766188.088188:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766188.088199:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766188/real 1775766188] req@ffff8b23864ffb80 x1862022098900736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766188.088212:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098900736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766188.088220:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766188.088224:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766188.088228:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098900736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766188.983576:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766190.007603:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766191.031666:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766192.055608:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766193.079649:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766193.207562:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766193.207570:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766193.207574:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766193.207582:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3654 00000100:00080000:3.0:1775766193.207591:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766193.207597:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766193.207633:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098900864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766193.207657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766193.207660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766193.207662:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766193.207664:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3654 00000100:00080000:3.0:1775766193.207666:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766193.207669:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766193.207676:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766193.207685:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098900992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766193.207695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766193.207696:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098900864:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766193.207698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766193.207701:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766193.207707:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3654 00000100:00080000:3.0:1775766193.207710:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766193.207712:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766193.207715:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766193.207724:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098901120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766193.207731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766193.207734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766193.207736:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766193.207738:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3654 00000100:00080000:3.0:1775766193.207741:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766193.207743:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766193.207745:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766193.207755:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098901248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766193.207778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766193.207780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766193.207782:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766193.207785:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3654 00000100:00080000:3.0:1775766193.207790:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775766193.207791:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766193.207797:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766193.207803:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766193.207825:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098901376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766193.207832:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766193/real 1775766193] req@ffff8b23864f0000 x1862022098900864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766193.207855:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098900864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766193.207864:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766193.207867:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766193.207870:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098900864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766193.207895:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098900992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766193.207912:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766193.207921:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766193/real 1775766193] req@ffff8b23864f0700 x1862022098900992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766193.207929:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098900992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766193.207934:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766193.207936:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766193.207939:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098900992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766193.207942:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098901120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766193.207953:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766193.207958:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766193/real 1775766193] req@ffff8b23864f1180 x1862022098901120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766193.207966:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098901120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766193.207971:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766193.207972:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766193.207974:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098901120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766193.207978:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098901248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766193.207991:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766193.207997:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766193/real 1775766193] req@ffff8b23864f0a80 x1862022098901248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766193.208004:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098901248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766193.208009:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766193.208011:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766193.208013:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098901248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766193.208016:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098901376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766193.208025:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766193.208030:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766193/real 1775766193] req@ffff8b23864f2680 x1862022098901376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766193.208037:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098901376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766193.208042:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766193.208044:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766193.208046:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098901376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766194.103583:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766195.127691:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766196.151579:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766197.175655:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766198.199598:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766198.327630:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766198.327640:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766198.327645:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766198.327664:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3659 00000100:00080000:1.0:1775766198.327689:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766198.327697:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766198.327777:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098901504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766198.327808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766198.327812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766198.327815:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766198.327819:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3659 00000100:00080000:1.0:1775766198.327823:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766198.327827:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766198.327837:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766198.327855:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098901504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775766198.327857:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098901632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766198.327875:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766198.327879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766198.327881:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766198.327885:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3659 00000100:00080000:1.0:1775766198.327889:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766198.327894:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766198.327900:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766198.327912:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098901760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766198.327923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775766198.327924:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775766198.327929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766198.327932:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766198.327953:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3659 00000100:00080000:1.0:1775766198.327959:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766198.327963:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766198.327967:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766198.327979:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098901888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766198.327988:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766198/real 1775766198] req@ffff8b23864ffb80 x1862022098901504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766198.327991:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766198.328001:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766198.328005:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766198.328023:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3659 00000100:00100000:0.0:1775766198.328027:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098901504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766198.328027:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766198.328035:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:0.0:1775766198.328044:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1775766198.328045:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775766198.328053:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766198.328057:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098901504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775766198.328064:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098902016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766198.328092:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098901632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766198.328114:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766198.328124:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766198/real 1775766198] req@ffff8b23864fe300 x1862022098901632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766198.328132:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098901632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766198.328138:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766198.328141:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766198.328143:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098901632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766198.328147:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098901760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766198.328158:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766198.328164:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766198/real 1775766198] req@ffff8b23864ff100 x1862022098901760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766198.328171:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098901760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766198.328177:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766198.328178:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766198.328180:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098901760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766198.328184:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098901888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766198.328195:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766198.328201:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766198/real 1775766198] req@ffff8b23864fd500 x1862022098901888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766198.328220:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098901888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766198.328225:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766198.328226:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766198.328228:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098901888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766198.328232:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098902016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766198.328242:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766198.328247:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766198/real 1775766198] req@ffff8b23864fdc00 x1862022098902016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766198.328254:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098902016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766198.328259:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766198.328261:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766198.328263:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098902016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766199.223621:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766200.247636:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766201.271721:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766202.295735:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766203.319705:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766203.447572:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766203.447596:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766203.447602:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766203.447614:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3664 00000100:00080000:2.0:1775766203.447629:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766203.447636:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766203.447735:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022098902144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766203.447779:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766203.447788:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766203.447789:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766203.447792:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3664 00000100:00080000:2.0:1775766203.447795:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766203.447799:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766203.447810:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766203.447822:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098902272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766203.447832:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766203.447838:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775766203.447839:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098902144:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775766203.447841:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766203.447849:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3664 00000100:00080000:2.0:1775766203.447852:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766203.447855:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766203.447858:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766203.447896:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098902400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766203.447904:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766203.447908:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766203.447910:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766203.447913:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3664 00000100:00080000:2.0:1775766203.447916:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766203.447919:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766203.447924:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766203.447933:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098902528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766203.447941:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766203.447944:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766203.447946:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766203.447949:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3664 00000100:00080000:2.0:1775766203.447952:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766203.447955:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775766203.447961:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:2.0:1775766203.447962:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766203.447975:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098902656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766203.448037:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766203/real 1775766203] req@ffff8b23b9ea6300 x1862022098902144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766203.448054:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022098902144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766203.448068:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766203.448071:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766203.448075:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098902144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766203.448098:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098902272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766203.448116:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766203.448125:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766203/real 1775766203] req@ffff8b23b9ea7480 x1862022098902272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766203.448133:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022098902272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766203.448139:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766203.448141:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766203.448143:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098902272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766203.448147:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098902400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766203.448158:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766203.448164:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766203/real 1775766203] req@ffff8b23b9ea4e00 x1862022098902400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766203.448171:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022098902400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766203.448177:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766203.448179:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766203.448181:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098902400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766203.448185:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098902528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766203.448198:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766203.448204:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766203/real 1775766203] req@ffff8b23b9ea4380 x1862022098902528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766203.448212:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022098902528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766203.448217:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766203.448219:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766203.448221:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098902528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766203.448224:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098902656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766203.448234:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766203.448240:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766203/real 1775766203] req@ffff8b23b9ea7100 x1862022098902656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766203.448247:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022098902656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766203.448252:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766203.448253:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766203.448255:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098902656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766204.343684:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766205.367702:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766206.391684:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766207.415717:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766208.439813:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766208.567577:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766208.567591:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766208.567596:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766208.567608:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3669 00000100:00080000:2.0:1775766208.567672:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766208.567682:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766208.567721:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098902784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766208.567751:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766208.567755:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766208.567757:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766208.567760:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3669 00000100:00080000:2.0:1775766208.567764:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766208.567767:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766208.567776:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766208.567786:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022098902912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766208.567798:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766208.567802:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766208.567803:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766208.567808:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3669 00000100:00080000:2.0:1775766208.567811:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766208.567815:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766208.567819:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766208.567828:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098903040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766208.567838:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766208.567841:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766208.567845:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766208.567853:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3669 00000100:00100000:0.0:1775766208.567854:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098902784:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775766208.567856:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766208.567864:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766208.567879:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766208.567891:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098903168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766208.567900:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766208.567902:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766208.567905:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766208.567908:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3669 00000100:00080000:2.0:1775766208.567911:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766208.567913:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766208.567917:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766208.567927:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098903296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766208.567979:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766208.568042:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766208/real 1775766208] req@ffff8b23b9ea4a80 x1862022098902784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766208.568067:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022098902784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766208.568080:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766208.568085:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766208.568090:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098902784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766208.568098:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098902912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766208.568123:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766208.568154:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766208/real 1775766208] req@ffff8b23b9ea5880 x1862022098902912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766208.568168:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022098902912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766208.568178:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766208.568182:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766208.568186:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098902912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766208.568221:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098903040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766208.568242:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766208.568253:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766208/real 1775766208] req@ffff8b23b9ea4000 x1862022098903040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766208.568266:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022098903040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766208.568297:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766208.568302:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766208.568306:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098903040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766208.568313:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098903168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766208.568337:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766208.568348:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766208/real 1775766208] req@ffff8b23b9ea6a00 x1862022098903168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766208.568361:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022098903168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766208.568370:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766208.568374:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766208.568378:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098903168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766208.568384:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098903296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766208.568403:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766208.568412:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766208/real 1775766208] req@ffff8b23b9ea4700 x1862022098903296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766208.568424:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022098903296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766208.568433:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766208.568437:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766208.568441:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098903296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766209.463701:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766210.487664:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766211.511641:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766212.535666:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766213.559682:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766213.687619:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766213.687627:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766213.687631:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766213.687640:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3675 00000100:00080000:3.0:1775766213.687651:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766213.687657:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766213.687695:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098903424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766213.687729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766213.687734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766213.687736:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766213.687741:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3675 00000100:00080000:3.0:1775766213.687745:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766213.687750:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766213.687760:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766213.687779:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098903552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766213.687791:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098903424:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766213.687797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766213.687803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766213.687805:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766213.687811:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3675 00000100:00080000:3.0:1775766213.687816:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766213.687821:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766213.687827:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766213.687839:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098903680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766213.687863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766213.687865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766213.687867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766213.687871:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3675 00000800:00000100:0.0:1775766213.687875:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766213.687877:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766213.687882:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766213.687886:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766213.687899:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098903808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766213.687909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766213.687914:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1775766213.687915:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766213/real 1775766213] req@ffff8b23864f1c00 x1862022098903424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766213.687919:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766213.687930:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3675 00000100:00080000:3.0:1775766213.687935:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766213.687939:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098903424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766213.687943:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:0.0:1775766213.687954:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1775766213.687954:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775766213.687960:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766213.687963:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098903424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766213.687968:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098903936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766213.687994:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098903552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766213.688013:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766213.688022:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766213/real 1775766213] req@ffff8b23864f0380 x1862022098903552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766213.688031:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098903552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766213.688036:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766213.688039:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766213.688041:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098903552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766213.688045:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098903680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766213.688056:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766213.688062:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766213/real 1775766213] req@ffff8b23864f1f80 x1862022098903680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766213.688070:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098903680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766213.688075:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766213.688077:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766213.688079:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098903680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766213.688082:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098903808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766213.688094:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766213.688099:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766213/real 1775766213] req@ffff8b23864f2680 x1862022098903808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766213.688123:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098903808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766213.688130:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766213.688132:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766213.688135:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098903808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766213.688139:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098903936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766213.688154:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766213.688161:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766213/real 1775766213] req@ffff8b23864f0a80 x1862022098903936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766213.688170:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098903936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766213.688177:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766213.688180:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766213.688182:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098903936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766214.583653:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766215.607670:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766216.631671:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766217.655688:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766218.679641:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766218.807710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766218.807722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766218.807728:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766218.807751:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3680 00000100:00080000:1.0:1775766218.807766:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766218.807787:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766218.807827:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098904064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766218.807863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766218.807867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766218.807869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766218.807873:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3680 00000100:00080000:1.0:1775766218.807877:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766218.807881:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766218.807889:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766218.807902:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098904192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766218.807913:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766218.807916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766218.807918:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766218.807922:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3680 00000100:00080000:1.0:1775766218.807924:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766218.807933:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766218.807934:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098904064:192.168.202.128@tcp:250:kworker.0 00000008:00100000:1.0:1775766218.807937:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766218.807955:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098904320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766218.807964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766218.807968:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766218.807970:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766218.807973:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3680 00000100:00080000:1.0:1775766218.807977:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766218.807980:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766218.807985:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766218.807994:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098904448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766218.808003:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766218.808005:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766218.808007:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766218.808010:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3680 00000100:00080000:1.0:1775766218.808013:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775766218.808020:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775766218.808021:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766218.808029:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766218.808039:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098904576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766218.808075:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766218/real 1775766218] req@ffff8b23864fea00 x1862022098904064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766218.808095:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098904064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766218.808104:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766218.808108:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766218.808112:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098904064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766218.808119:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098904192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766218.808155:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766218.808167:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766218/real 1775766218] req@ffff8b23864fd180 x1862022098904192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766218.808178:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098904192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766218.808185:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766218.808188:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766218.808191:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098904192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766218.808215:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098904320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766218.808231:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766218.808257:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766218/real 1775766218] req@ffff8b23864fce00 x1862022098904320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766218.808269:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098904320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766218.808277:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766218.808280:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766218.808283:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098904320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766218.808289:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098904448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766218.808307:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766218.808316:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766218/real 1775766218] req@ffff8b23864fe680 x1862022098904448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766218.808327:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098904448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766218.808335:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766218.808337:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766218.808340:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098904448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766218.808345:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098904576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766218.808359:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766218.808384:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766218/real 1775766218] req@ffff8b23864fca80 x1862022098904576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766218.808399:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098904576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766218.808413:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766218.808419:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766218.808425:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098904576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766219.703936:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766220.727654:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766221.751669:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766222.775694:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766223.799662:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766223.927589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766223.927597:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766223.927601:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766223.927609:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3685 00000100:00080000:3.0:1775766223.927619:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766223.927624:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766223.927656:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098904704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766223.927678:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766223.927681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766223.927682:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766223.927685:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3685 00000100:00080000:3.0:1775766223.927687:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766223.927689:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766223.927696:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766223.927705:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098904832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766223.927713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766223.927716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766223.927717:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766223.927720:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3685 00000100:00080000:3.0:1775766223.927723:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766223.927725:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766223.927729:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766223.927736:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098904704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766223.927738:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098904960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766223.927752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766223.927754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766223.927755:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766223.927758:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3685 00000100:00080000:3.0:1775766223.927759:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766223.927761:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766223.927764:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766223.927770:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098905088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766223.927775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766223.927777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766223.927778:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766223.927781:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3685 00000100:00080000:3.0:1775766223.927782:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766223.927784:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766223.927786:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766223.927792:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098905216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766223.927845:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766223.927887:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766223/real 1775766223] req@ffff8b23864f1180 x1862022098904704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766223.927902:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098904704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766223.927910:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766223.927913:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766223.927916:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098904704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766223.927922:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098904832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766223.927940:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766223.927948:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766223/real 1775766223] req@ffff8b23864f0700 x1862022098904832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766223.927956:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098904832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766223.927965:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766223.927968:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766223.927970:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098904832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766223.927988:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098904960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766223.927998:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766223.928004:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766223/real 1775766223] req@ffff8b23864f0000 x1862022098904960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766223.928011:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098904960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766223.928016:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766223.928019:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766223.928020:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098904960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766223.928024:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098905088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766223.928058:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766223.928065:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766223/real 1775766223] req@ffff8b23864f3b80 x1862022098905088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766223.928073:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098905088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766223.928078:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766223.928080:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766223.928082:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098905088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766223.928087:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098905216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766223.928098:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766223.928103:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766223/real 1775766223] req@ffff8b23864f3100 x1862022098905216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766223.928110:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098905216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766223.928115:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766223.928117:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766223.928118:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098905216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766224.823607:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766225.847638:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766226.871568:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766227.895634:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766228.919615:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766229.047559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766229.047567:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766229.047570:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766229.047579:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3690 00000100:00080000:3.0:1775766229.047589:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766229.047607:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766229.047651:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098905344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766229.047674:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766229.047676:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766229.047678:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766229.047680:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3690 00000100:00080000:3.0:1775766229.047682:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766229.047685:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766229.047692:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766229.047702:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098905472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766229.047712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766229.047715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775766229.047718:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098905344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766229.047718:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766229.047723:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3690 00000100:00080000:3.0:1775766229.047728:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766229.047731:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766229.047735:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766229.047743:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098905600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766229.047750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766229.047752:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766229.047754:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766229.047757:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3690 00000100:00080000:3.0:1775766229.047760:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766229.047762:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766229.047765:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766229.047771:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098905728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766229.047778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775766229.047783:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766229.047784:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766229.047786:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766229.047789:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3690 00000100:00080000:3.0:1775766229.047791:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766229.047793:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766229.047796:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766229.047803:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098905856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766229.047830:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766229/real 1775766229] req@ffff8b23864f2a00 x1862022098905344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766229.047845:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098905344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766229.047853:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766229.047856:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766229.047859:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098905344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766229.047878:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098905472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766229.047893:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766229.047902:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766229/real 1775766229] req@ffff8b23864f2d80 x1862022098905472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766229.047910:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098905472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766229.047915:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766229.047917:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766229.047920:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098905472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766229.047923:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098905600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766229.047934:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766229.047940:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766229/real 1775766229] req@ffff8b23864f3480 x1862022098905600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766229.047947:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098905600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766229.047952:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766229.047954:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766229.047956:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098905600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766229.047959:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098905728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766229.047969:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766229.047974:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766229/real 1775766229] req@ffff8b23864f3100 x1862022098905728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766229.047982:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098905728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766229.047987:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766229.047989:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766229.047991:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098905728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766229.047994:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098905856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766229.048004:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766229.048009:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766229/real 1775766229] req@ffff8b23864f3b80 x1862022098905856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766229.048016:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098905856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766229.048021:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766229.048023:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766229.048025:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098905856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766229.943689:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766230.967622:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766231.991685:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766233.015609:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766234.039615:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766234.167559:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766234.167570:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766234.167575:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766234.167585:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3695 00000100:00080000:0.0:1775766234.167597:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766234.167604:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766234.167647:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098905984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766234.167678:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766234.167683:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766234.167685:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766234.167689:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3695 00000100:00080000:0.0:1775766234.167693:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766234.167698:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766234.167708:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766234.167721:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098905984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766234.167725:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098906112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766234.167740:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766234.167743:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766234.167745:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766234.167750:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3695 00000100:00080000:0.0:1775766234.167755:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766234.167760:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766234.167765:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766234.167780:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098906240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766234.167782:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766234.167794:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766234.167799:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766234.167801:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766234.167806:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3695 00000100:00080000:0.0:1775766234.167810:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766234.167815:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766234.167820:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766234/real 1775766234] req@ffff8b2382dac000 x1862022098905984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775766234.167821:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766234.167840:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098906368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766234.167842:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098905984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766234.167871:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766234.167872:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766234.167877:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00080000:0.0:1775766234.167879:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1775766234.167882:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098905984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766234.167884:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766234.167892:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3695 00000100:00080000:0.0:1775766234.167896:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766234.167901:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766234.167906:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766234.167907:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098906112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766234.167925:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098906496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766234.167931:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766234.167942:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766234/real 1775766234] req@ffff8b2382daf100 x1862022098906112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766234.167953:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098906112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766234.167959:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766234.167962:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766234.167965:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098906112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766234.167969:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098906240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766234.167981:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766234.167986:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766234/real 1775766234] req@ffff8b2382daca80 x1862022098906240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766234.168000:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098906240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766234.168009:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766234.168012:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766234.168016:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098906240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766234.168021:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098906368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766234.168042:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766234.168053:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766234/real 1775766234] req@ffff8b2382dad880 x1862022098906368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766234.168065:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098906368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766234.168071:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766234.168073:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766234.168075:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098906368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766234.168088:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098906496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766234.168102:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766234.168108:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766234/real 1775766234] req@ffff8b2382dac000 x1862022098906496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766234.168115:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098906496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766234.168125:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766234.168127:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766234.168129:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098906496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766235.063584:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766236.087633:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766237.111562:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766238.135617:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766239.159610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766239.287586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766239.287595:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766239.287600:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766239.287610:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3700 00000100:00080000:1.0:1775766239.287620:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766239.287628:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766239.287666:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098906624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766239.287692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766239.287695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766239.287697:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766239.287700:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3700 00000100:00080000:1.0:1775766239.287703:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766239.287706:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766239.287714:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766239.287726:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098906752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766239.287737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766239.287742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1775766239.287742:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098906624:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766239.287745:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766239.287754:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3700 00000100:00080000:1.0:1775766239.287757:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766239.287760:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766239.287763:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766239.287773:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098906880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766239.287786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766239.287790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766239.287793:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766239.287798:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3700 00000100:00080000:1.0:1775766239.287805:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766239.287810:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766239.287818:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766239.287833:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766239.287872:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766239/real 1775766239] req@ffff8b23864ff800 x1862022098906624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766239.287892:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098906624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766239.287900:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098907008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766239.287903:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766239.287913:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766239.287921:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098906624:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766239.287929:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766239.287935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766239.287938:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766239.287944:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3700 00000100:00080000:1.0:1775766239.287949:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766239.287952:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766239.287958:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766239.287961:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098906752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:1.0:1775766239.287976:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098907136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766239.287990:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766239.288004:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766239/real 1775766239] req@ffff8b23864fc380 x1862022098906752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766239.288015:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098906752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766239.288028:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766239.288031:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766239.288033:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098906752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766239.288046:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098906880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766239.288061:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766239.288068:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766239/real 1775766239] req@ffff8b23864fed80 x1862022098906880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766239.288078:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098906880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766239.288085:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766239.288087:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766239.288090:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098906880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766239.288094:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098907008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766239.288114:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766239.288126:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766239/real 1775766239] req@ffff8b23864fc000 x1862022098907008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766239.288141:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098907008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766239.288152:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766239.288156:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766239.288161:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098907008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766239.288183:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098907136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766239.288203:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766239.288212:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766239/real 1775766239] req@ffff8b23864fdf80 x1862022098907136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766239.288221:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098907136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766239.288228:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766239.288230:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766239.288233:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098907136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766240.183672:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766241.207764:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766242.231701:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766243.255738:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766244.279695:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766244.407586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766244.407594:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766244.407598:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766244.407606:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3705 00000100:00080000:3.0:1775766244.407616:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766244.407621:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766244.407654:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098907264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766244.407678:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766244.407680:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766244.407682:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766244.407685:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3705 00000100:00080000:3.0:1775766244.407688:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766244.407691:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766244.407698:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766244.407706:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098907392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766244.407714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766244.407716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766244.407717:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766244.407720:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3705 00000100:00080000:3.0:1775766244.407722:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766244.407724:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766244.407727:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766244.407733:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098907520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766244.407741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766244.407743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766244.407745:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766244.407748:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3705 00000100:00080000:3.0:1775766244.407749:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766244.407752:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766244.407755:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766244.407761:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098907648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766244.407769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766244.407772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766244.407773:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766244.407776:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3705 00000100:00080000:3.0:1775766244.407777:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766244.407781:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766244.407784:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766244.407791:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098907776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766244.407793:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098907264:192.168.202.128@tcp:250:kworker.0 00000800:00000100:0.0:1775766244.407887:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766244.407935:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766244/real 1775766244] req@ffff8b23864f0000 x1862022098907264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766244.407952:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098907264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766244.407964:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766244.407970:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766244.407974:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098907264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766244.407985:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098907392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766244.408013:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766244.408027:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766244/real 1775766244] req@ffff8b23864f0700 x1862022098907392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766244.408041:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098907392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766244.408050:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766244.408054:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766244.408057:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098907392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766244.408064:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098907520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766244.408086:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766244.408097:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766244/real 1775766244] req@ffff8b23864f1180 x1862022098907520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766244.408110:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098907520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766244.408118:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766244.408122:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766244.408125:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098907520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766244.408157:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098907648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766244.408180:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766244.408190:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766244/real 1775766244] req@ffff8b23864f0a80 x1862022098907648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766244.408209:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098907648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766244.408218:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766244.408221:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766244.408224:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098907648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766244.408230:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098907776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766244.408251:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766244.408261:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766244/real 1775766244] req@ffff8b23864f2680 x1862022098907776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766244.408291:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098907776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766244.408300:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766244.408303:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766244.408306:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098907776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766245.303674:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766246.327698:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766247.351867:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766248.375643:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766249.399670:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766249.527574:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766249.527586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766249.527592:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766249.527604:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3710 00000100:00080000:3.0:1775766249.527617:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766249.527625:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766249.527674:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098907904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766249.527715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766249.527720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766249.527722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766249.527729:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3710 00000100:00080000:3.0:1775766249.527732:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766249.527737:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766249.527748:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766249.527763:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098908032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766249.527776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766249.527780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766249.527782:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766249.527786:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3710 00000100:00080000:3.0:1775766249.527789:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766249.527793:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766249.527798:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766249.527809:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098908160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766249.527843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766249.527856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1775766249.527863:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098907904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766249.527863:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766249.527878:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3710 00000100:00080000:3.0:1775766249.527887:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766249.527895:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766249.527905:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766249.527929:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098908288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766249.527946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766249.527950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766249.527952:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766249.527957:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3710 00000100:00080000:3.0:1775766249.527964:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766249.527972:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766249.527982:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766249.527992:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098908032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766249.528008:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098908416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766249.528032:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098908160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766249.528071:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098908288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766249.528096:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098908416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766250.423642:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766251.447616:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766252.471616:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775766252.599794:0:2364:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775766252.599836:0:2364:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766252.599852:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766252.599891:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766252.599898:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766252.599905:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766252.599911:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775766252.599957:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766249/real 1775766252] req@ffff8b23864f2680 x1862022098908288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766252.599985:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098908288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766252.599996:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766252.600001:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766252.600005:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098908288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766252.600012:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766249/real 1775766252] req@ffff8b23864f0a80 x1862022098908416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766252.600026:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098908416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766252.600046:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766252.600050:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766252.600055:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098908416:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766252.600064:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766249/real 1775766252] req@ffff8b23864f1f80 x1862022098907904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766252.600082:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098907904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766252.600092:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766252.600119:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766252.600124:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098907904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766252.600148:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766249/real 1775766252] req@ffff8b23864f0380 x1862022098908032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766252.600176:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098908032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766252.600185:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766252.600190:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766252.600211:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098908032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766252.600220:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766249/real 1775766252] req@ffff8b23864f1c00 x1862022098908160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766252.600236:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098908160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766252.600246:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766252.600252:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766252.600257:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098908160:192.168.202.128@tcp:38:kworker.0 00010000:00010000:3.0:1775766253.495588:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766254.519571:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766254.647620:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766254.647631:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766254.647636:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766254.647649:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3716 00000100:00080000:2.0:1775766254.647662:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766254.647686:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766254.647724:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098908544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766254.647759:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766254.647762:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766254.647764:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766254.647767:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3716 00000100:00080000:2.0:1775766254.647771:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766254.647774:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766254.647786:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766254.647803:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098908672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766254.647812:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098908544:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775766254.647827:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766254.647832:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766254.647834:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766254.647841:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3716 00000100:00080000:2.0:1775766254.647848:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766254.647854:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766254.647863:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766254.647882:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022098908800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766254.647895:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766254.647899:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1775766254.647904:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775766254.647905:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766254.647916:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3716 00000100:00080000:2.0:1775766254.647923:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766254.647928:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766254.647935:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766254.647953:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098908928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766254.647961:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766254/real 1775766254] req@ffff8b23b9ea5500 x1862022098908544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766254.647973:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766254.647989:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766254.647992:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766254.648000:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3716 00000100:00100000:0.0:1775766254.648005:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022098908544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766254.648008:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766254.648020:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:0.0:1775766254.648024:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1775766254.648030:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775766254.648034:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766254.648042:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098908544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775766254.648052:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098909056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766254.648101:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098908672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766254.648136:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766254.648154:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766254/real 1775766254] req@ffff8b23b9ea5c00 x1862022098908672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766254.648168:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022098908672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766254.648197:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766254.648201:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766254.648206:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098908672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766254.648214:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098908800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766254.648244:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766254.648258:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766254/real 1775766254] req@ffff8b23b9ea6d80 x1862022098908800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766254.648272:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022098908800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766254.648283:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766254.648287:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766254.648299:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098908800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766254.648308:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098908928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766254.648339:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766254.648355:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766254/real 1775766254] req@ffff8b23b9ea6680 x1862022098908928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766254.648372:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022098908928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766254.648382:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766254.648386:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766254.648391:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098908928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766254.648398:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098909056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766254.648426:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766254.648440:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766254/real 1775766254] req@ffff8b23b9ea5f80 x1862022098909056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766254.648496:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022098909056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766254.648507:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766254.648511:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766254.648515:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098909056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766255.543760:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766256.567586:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766257.591595:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766258.615610:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766259.639645:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766259.767658:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766259.767671:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766259.767676:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766259.767688:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3721 00000100:00080000:1.0:1775766259.767702:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766259.767710:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766259.767763:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098909184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766259.767801:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766259.767806:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766259.767808:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766259.767813:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3721 00000100:00080000:1.0:1775766259.767817:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766259.767834:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766259.767852:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766259.767877:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098909312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766259.767902:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766259.767908:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766259.767912:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766259.767927:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3721 00000100:00100000:0.0:1775766259.767928:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098909184:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766259.767938:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766259.767953:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766259.767964:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766259.767993:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098909440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766259.768007:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766259.768012:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766259.768014:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766259.768020:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3721 00000100:00080000:1.0:1775766259.768041:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766259.768046:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766259.768053:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766259.768059:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766259.768076:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098909568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766259.768092:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766259.768096:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766259.768099:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766259.768103:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3721 00000100:00080000:1.0:1775766259.768107:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766259.768112:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766259.768118:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766259.768165:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098909696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766259.768174:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766259/real 1775766259] req@ffff8b23864fdf80 x1862022098909184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766259.768240:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098909184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766259.768256:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766259.768262:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766259.768268:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098909184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766259.768310:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098909312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766259.768344:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766259.768361:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766259/real 1775766259] req@ffff8b23864fc000 x1862022098909312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766259.768377:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098909312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766259.768387:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766259.768401:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766259.768405:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098909312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766259.768413:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098909440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766259.768433:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766259.768443:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766259/real 1775766259] req@ffff8b23864fed80 x1862022098909440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766259.768542:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098909440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766259.768574:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766259.768581:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766259.768586:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098909440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766259.768596:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098909568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766259.768633:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766259.768649:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766259/real 1775766259] req@ffff8b23864fc380 x1862022098909568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766259.768664:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098909568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766259.768674:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766259.768679:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766259.768683:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098909568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766259.768693:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098909696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766259.768729:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766259.768750:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766259/real 1775766259] req@ffff8b23864ff800 x1862022098909696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766259.768773:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098909696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766259.768791:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766259.768798:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766259.768806:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098909696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766260.663639:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766261.687628:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766262.711621:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766263.735625:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766264.759616:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766264.887517:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766264.887525:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766264.887529:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766264.887537:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3726 00000100:00080000:1.0:1775766264.887546:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766264.887552:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766264.887588:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098909824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766264.887628:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766264.887631:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766264.887633:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766264.887636:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3726 00000100:00080000:1.0:1775766264.887638:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766264.887640:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766264.887649:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766264.887662:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098909952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766264.887671:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766264.887675:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098909824:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766264.887675:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766264.887678:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766264.887686:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3726 00000100:00080000:1.0:1775766264.887688:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766264.887691:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766264.887694:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766264.887705:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098910080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766264.887712:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766264.887715:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766264.887717:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766264.887719:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3726 00000100:00080000:1.0:1775766264.887729:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766264.887732:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766264.887735:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766264.887739:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766264.887744:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098910208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766264.887750:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766264.887753:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766264.887755:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766264.887758:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3726 00000100:00080000:1.0:1775766264.887761:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766264.887763:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766264.887767:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766264.887792:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098910336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766264.887807:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766264/real 1775766264] req@ffff8b23864fca80 x1862022098909824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766264.887823:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098909824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766264.887831:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766264.887834:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766264.887838:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098909824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766264.887857:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098909952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766264.887874:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766264.887882:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766264/real 1775766264] req@ffff8b23864fe680 x1862022098909952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766264.887890:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098909952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766264.887895:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766264.887898:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766264.887900:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098909952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766264.887904:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098910080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766264.887914:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766264.887921:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766264/real 1775766264] req@ffff8b23864fce00 x1862022098910080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766264.887928:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098910080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766264.887933:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766264.887935:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766264.887937:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098910080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766264.887940:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098910208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766264.887952:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766264.887957:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766264/real 1775766264] req@ffff8b23864fd180 x1862022098910208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766264.887965:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098910208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766264.887969:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766264.887971:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766264.887973:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098910208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766264.887976:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098910336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766264.887986:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766264.888008:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766264/real 1775766264] req@ffff8b23864fea00 x1862022098910336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766264.888017:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098910336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766264.888023:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766264.888026:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766264.888029:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098910336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766265.783590:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766266.807694:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766267.831656:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766268.855729:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766269.879632:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766270.007571:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766270.007581:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766270.007586:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766270.007595:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3731 00000100:00080000:1.0:1775766270.007606:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766270.007613:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766270.007679:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098910464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766270.007729:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766270.007734:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766270.007737:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766270.007742:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3731 00000100:00080000:1.0:1775766270.007747:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766270.007753:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766270.007766:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766270.007801:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098910592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766270.007807:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098910464:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766270.007834:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766270.007839:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766270.007842:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766270.007866:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3731 00000100:00080000:1.0:1775766270.007872:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766270.007877:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766270.007885:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766270.007904:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098910720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766270.007907:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775766270.007933:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766270.007936:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766270.007939:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766270.007945:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3731 00000100:00080000:1.0:1775766270.007951:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766270.007958:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766270/real 1775766270] req@ffff8b23864fdc00 x1862022098910464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766270.007958:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766270.007969:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766270.007991:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098910848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766270.007996:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098910464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766270.008013:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766270.008052:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766270.008053:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766270.008056:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3731 00000100:00080000:1.0:1775766270.008059:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766270.008061:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766270.008065:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766270.008073:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098910976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766270.008194:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766270.008200:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766270.008204:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098910464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766270.008234:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098910592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766270.008253:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766270.008263:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766270/real 1775766270] req@ffff8b23864fd500 x1862022098910592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766270.008271:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098910592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766270.008277:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766270.008279:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766270.008281:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098910592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766270.008286:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098910720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766270.008297:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766270.008303:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766270/real 1775766270] req@ffff8b23864ff100 x1862022098910720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766270.008310:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098910720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766270.008315:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766270.008317:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766270.008318:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098910720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766270.008322:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098910848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766270.008333:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766270.008338:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766270/real 1775766270] req@ffff8b23864fe300 x1862022098910848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766270.008346:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098910848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766270.008350:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766270.008352:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766270.008354:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098910848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766270.008357:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098910976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766270.008367:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766270.008372:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766270/real 1775766270] req@ffff8b23864ffb80 x1862022098910976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766270.008379:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098910976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766270.008384:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766270.008385:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766270.008387:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098910976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766270.903645:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766271.927604:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766272.951660:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766273.975587:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766274.999596:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766275.127563:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766275.127571:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766275.127575:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766275.127583:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3736 00000100:00080000:3.0:1775766275.127593:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766275.127599:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766275.127643:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098911104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766275.127664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766275.127666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766275.127667:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766275.127670:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3736 00000100:00080000:3.0:1775766275.127672:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766275.127675:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766275.127682:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766275.127692:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098911232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766275.127700:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766275.127703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766275.127704:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766275.127708:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3736 00000100:00080000:3.0:1775766275.127710:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766275.127714:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766275.127715:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098911104:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775766275.127717:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766275.127728:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098911360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766275.127744:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766275.127747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766275.127749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766275.127751:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3736 00000100:00080000:3.0:1775766275.127753:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766275.127756:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766275.127758:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766275.127765:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098911488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766275.127770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766275.127772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766275.127774:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766275.127776:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3736 00000100:00080000:3.0:1775766275.127779:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766275.127781:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766275.127784:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766275.127791:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098911616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766275.127799:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766275.127834:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766275/real 1775766275] req@ffff8b23864f1180 x1862022098911104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766275.127849:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098911104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766275.127857:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766275.127860:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766275.127863:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098911104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766275.127868:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098911232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766275.127883:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766275.127890:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766275/real 1775766275] req@ffff8b23864f0700 x1862022098911232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766275.127897:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098911232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766275.127907:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766275.127909:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766275.127911:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098911232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766275.127929:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098911360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766275.127940:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766275.127945:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766275/real 1775766275] req@ffff8b23864f0000 x1862022098911360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766275.127952:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098911360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766275.127958:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766275.127959:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766275.127961:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098911360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766275.127965:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098911488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766275.127975:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766275.127999:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766275/real 1775766275] req@ffff8b23864f3b80 x1862022098911488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766275.128023:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098911488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766275.128028:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766275.128030:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766275.128033:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098911488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766275.128037:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098911616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766275.128050:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766275.128057:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766275/real 1775766275] req@ffff8b23864f3100 x1862022098911616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766275.128064:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098911616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766275.128069:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766275.128070:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766275.128072:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098911616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766276.023643:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766277.047651:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766278.071565:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766279.095695:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766280.119630:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766280.247634:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766280.247657:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766280.247662:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766280.247674:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3741 00000100:00080000:1.0:1775766280.247688:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766280.247695:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766280.247753:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098911744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766280.247793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766280.247797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766280.247798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766280.247802:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3741 00000100:00080000:1.0:1775766280.247804:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766280.247807:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766280.247816:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766280.247825:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098911872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766280.247835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766280.247866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766280.247869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766280.247875:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3741 00000100:00080000:1.0:1775766280.247880:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766280.247886:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766280.247891:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766280.247915:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098912000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766280.247948:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098911744:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766280.247951:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766280.247963:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766280.247968:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766280.247975:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3741 00000100:00080000:1.0:1775766280.247978:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766280.247982:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766280.247987:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766280.248007:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098912128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766280.248016:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766280.248020:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766280.248022:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766280.248024:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3741 00000100:00080000:1.0:1775766280.248034:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766280.248036:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766280.248040:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766280.248046:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766280.248051:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098912256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766280.248095:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766280/real 1775766280] req@ffff8b23864ffb80 x1862022098911744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766280.248112:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098911744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766280.248121:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766280.248124:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766280.248127:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098911744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766280.248135:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098911872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766280.248162:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766280.248177:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766280/real 1775766280] req@ffff8b23864fe300 x1862022098911872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766280.248222:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098911872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766280.248229:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766280.248232:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766280.248236:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098911872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766280.248286:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098912000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766280.248332:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766280.248346:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766280/real 1775766280] req@ffff8b23864ff100 x1862022098912000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766280.248357:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098912000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766280.248363:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766280.248366:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766280.248368:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098912000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766280.248374:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098912128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766280.248391:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766280.248398:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766280/real 1775766280] req@ffff8b23864fd500 x1862022098912128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766280.248407:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098912128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766280.248429:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766280.248431:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766280.248434:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098912128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766280.248438:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098912256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766280.248450:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766280.248456:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766280/real 1775766280] req@ffff8b23864fdc00 x1862022098912256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766280.248464:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098912256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766280.248469:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766280.248471:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766280.248490:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098912256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766281.143706:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766282.167808:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766283.191695:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766284.215710:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766285.239734:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766285.367620:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766285.367631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766285.367636:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766285.367648:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3746 00000100:00080000:3.0:1775766285.367661:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766285.367679:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766285.367731:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098912384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766285.367765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766285.367769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766285.367772:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766285.367776:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3746 00000100:00080000:3.0:1775766285.367781:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766285.367786:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766285.367799:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766285.367818:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098912512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766285.367820:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098912384:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766285.367836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766285.367839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766285.367841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766285.367845:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3746 00000100:00080000:3.0:1775766285.367849:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766285.367852:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766285.367856:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766285.367866:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098912640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766285.367873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766285.367877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766285.367878:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766285.367894:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3746 00000100:00080000:3.0:1775766285.367898:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766285.367901:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766285.367905:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766285.367914:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098912768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766285.367922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775766285.367927:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766285.367928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766285.367931:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766285.367935:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3746 00000100:00080000:3.0:1775766285.367938:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766285.367941:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766285.367944:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766285.367953:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098912896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766285.367979:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766285/real 1775766285] req@ffff8b23864f3480 x1862022098912384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766285.367996:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098912384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766285.368005:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766285.368009:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766285.368012:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098912384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766285.368036:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098912512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766285.368054:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766285.368063:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766285/real 1775766285] req@ffff8b23864f2d80 x1862022098912512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766285.368072:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098912512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766285.368077:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766285.368080:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766285.368082:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098912512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766285.368088:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098912640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766285.368106:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766285.368117:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766285/real 1775766285] req@ffff8b23864f2a00 x1862022098912640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766285.368130:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098912640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766285.368140:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766285.368144:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766285.368147:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098912640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766285.368154:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098912768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766285.368172:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766285.368180:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766285/real 1775766285] req@ffff8b23864f3100 x1862022098912768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766285.368191:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098912768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766285.368200:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766285.368203:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766285.368206:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098912768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766285.368213:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098912896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766285.368234:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766285.368246:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766285/real 1775766285] req@ffff8b23864f3b80 x1862022098912896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766285.368255:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098912896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766285.368261:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766285.368263:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766285.368265:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098912896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766286.263669:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766287.287638:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766288.311769:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766289.335673:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766290.359694:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766290.487593:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766290.487606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766290.487611:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766290.487624:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3751 00000100:00080000:0.0:1775766290.487639:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766290.487647:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766290.487691:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098913024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766290.487727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766290.487731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766290.487733:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766290.487737:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3751 00000100:00080000:0.0:1775766290.487739:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766290.487742:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766290.487752:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766290.487763:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098913152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766290.487772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766290.487776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766290.487777:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766290.487781:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3751 00000100:00080000:0.0:1775766290.487783:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766290.487786:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766290.487790:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766290.487798:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098913280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766290.487806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766290.487810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766290.487811:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766290.487815:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3751 00000100:00080000:0.0:1775766290.487818:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766290.487821:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766290.487827:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766290.487844:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098913024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766290.487866:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098913408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766290.487880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766290.487885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766290.487887:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766290.487893:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3751 00000100:00080000:0.0:1775766290.487899:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766290.487904:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766290.487912:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766290.487931:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098913536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766290.487946:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766290.488009:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766290/real 1775766290] req@ffff8b2382daed80 x1862022098913024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766290.488036:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098913024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766290.488046:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766290.488050:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766290.488054:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098913024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766290.488062:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098913152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766290.488086:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766290.488096:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766290/real 1775766290] req@ffff8b2382dac380 x1862022098913152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766290.488106:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098913152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766290.488113:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766290.488116:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766290.488119:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098913152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766290.488123:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098913280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766290.488136:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766290.488143:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766290/real 1775766290] req@ffff8b2382dae680 x1862022098913280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766290.488153:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098913280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766290.488165:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766290.488167:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766290.488170:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098913280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766290.488202:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098913408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766290.488216:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766290.488224:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766290/real 1775766290] req@ffff8b2382dad500 x1862022098913408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766290.488232:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098913408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766290.488239:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766290.488241:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766290.488243:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098913408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766290.488248:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098913536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766290.488260:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766290.488267:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766290/real 1775766290] req@ffff8b2382daea00 x1862022098913536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766290.488286:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098913536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766290.488293:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766290.488296:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766290.488299:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098913536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766291.383632:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766292.407638:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766293.431609:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766294.455644:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766295.479702:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766295.607709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766295.607735:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766295.607742:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766295.607759:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3757 00000100:00080000:3.0:1775766295.607778:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766295.607815:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766295.607906:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098913664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766295.607975:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766295.607984:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766295.607988:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766295.607997:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3757 00000100:00080000:3.0:1775766295.608016:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766295.608025:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766295.608043:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766295.608066:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098913792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766295.608084:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766295.608090:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766295.608093:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766295.608107:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3757 00000100:00100000:0.0:1775766295.608109:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098913664:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766295.608114:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766295.608128:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766295.608135:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766295.608153:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098913920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766295.608167:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766295.608172:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766295.608175:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766295.608180:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3757 00000100:00080000:3.0:1775766295.608186:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766295.608189:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766295.608195:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766295.608208:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098914048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766295.608218:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766295.608223:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766295.608229:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775766295.608230:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766295.608243:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3757 00000100:00080000:3.0:1775766295.608250:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766295.608257:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766295.608263:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766295.608277:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098914176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766295.608315:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766295/real 1775766295] req@ffff8b23864f0000 x1862022098913664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766295.608348:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098913664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766295.608363:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766295.608368:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766295.608374:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098913664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766295.608384:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098913792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766295.608412:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766295.608426:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766295/real 1775766295] req@ffff8b23864f0700 x1862022098913792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766295.608441:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098913792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766295.608485:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766295.608491:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766295.608496:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098913792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766295.608531:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098913920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766295.608556:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766295.608569:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766295/real 1775766295] req@ffff8b23864f1180 x1862022098913920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766295.608584:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098913920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766295.608605:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766295.608609:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766295.608613:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098913920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766295.608620:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098914048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766295.608640:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766295.608650:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766295/real 1775766295] req@ffff8b23864f1c00 x1862022098914048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766295.608664:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098914048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766295.608674:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766295.608677:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766295.608681:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098914048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766295.608687:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098914176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766295.608719:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766295.608729:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766295/real 1775766295] req@ffff8b23864f0380 x1862022098914176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766295.608743:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098914176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766295.608753:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766295.608756:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766295.608760:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098914176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766296.503703:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766297.527628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766298.551826:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766299.575719:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766300.599656:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766300.727514:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766300.727522:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766300.727527:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766300.727535:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3762 00000100:00080000:3.0:1775766300.727544:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766300.727550:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766300.727587:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098914304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766300.727612:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766300.727615:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766300.727616:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766300.727619:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3762 00000100:00080000:3.0:1775766300.727621:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766300.727623:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766300.727631:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766300.727640:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098914432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766300.727647:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766300.727650:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766300.727651:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766300.727654:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3762 00000100:00080000:3.0:1775766300.727672:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766300.727675:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766300.727678:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766300.727685:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098914560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766300.727692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766300.727694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766300.727695:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766300.727699:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3762 00000100:00080000:3.0:1775766300.727701:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766300.727703:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766300.727706:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766300.727708:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098914304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766300.727715:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098914688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766300.727722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766300.727724:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766300.727725:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766300.727727:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3762 00000100:00080000:3.0:1775766300.727729:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766300.727731:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766300.727745:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766300.727754:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098914816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766300.727806:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766300.727865:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766300/real 1775766300] req@ffff8b23864f1f80 x1862022098914304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766300.727881:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098914304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766300.727894:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766300.727897:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766300.727900:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098914304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766300.727906:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098914432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766300.727923:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766300.727931:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766300/real 1775766300] req@ffff8b23864f0a80 x1862022098914432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766300.727958:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098914432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766300.727967:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766300.727970:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766300.727973:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098914432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766300.727979:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098914560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766300.728001:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766300.728029:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766300/real 1775766300] req@ffff8b23864f2680 x1862022098914560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766300.728041:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098914560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766300.728048:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766300.728050:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766300.728052:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098914560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766300.728074:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098914688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766300.728089:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766300.728097:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766300/real 1775766300] req@ffff8b23864f0380 x1862022098914688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766300.728104:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098914688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766300.728110:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766300.728112:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766300.728114:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098914688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766300.728117:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098914816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766300.728127:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766300.728132:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766300/real 1775766300] req@ffff8b23864f1c00 x1862022098914816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766300.728139:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098914816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766300.728145:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766300.728146:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766300.728148:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098914816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766301.623780:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766302.647637:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766303.671591:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766304.695689:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766305.719731:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766305.847758:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766305.847774:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766305.847781:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766305.847812:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3767 00000100:00080000:1.0:1775766305.847831:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766305.847842:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766305.847911:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098914944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766305.847951:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766305.847955:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766305.847957:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766305.847962:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3767 00000100:00080000:1.0:1775766305.847967:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766305.847972:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766305.847984:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766305.848000:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098915072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766305.848015:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098914944:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766305.848041:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766305.848044:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766305.848047:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766305.848054:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3767 00000100:00080000:1.0:1775766305.848059:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766305.848064:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766305.848070:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766305.848087:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098915200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766305.848098:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766305.848102:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766305.848105:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766305.848112:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3767 00000800:00000100:0.0:1775766305.848114:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775766305.848126:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766305.848133:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766305.848141:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766305.848165:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098915328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766305.848167:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766305/real 1775766305] req@ffff8b23864fea00 x1862022098914944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766305.848242:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766305.848245:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766305.848248:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766305.848252:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098914944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766305.848257:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3767 00000100:00080000:1.0:1775766305.848268:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766305.848284:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:0.0:1775766305.848285:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766305.848298:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000008:00100000:1.0:1775766305.848299:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766305.848310:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098914944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775766305.848322:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098915456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766305.848341:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098915072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766305.848376:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766305.848392:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766305/real 1775766305] req@ffff8b23864fd180 x1862022098915072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766305.848409:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098915072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766305.848420:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766305.848425:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766305.848429:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098915072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766305.848439:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098915200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766305.848490:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766305.848501:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766305/real 1775766305] req@ffff8b23864fce00 x1862022098915200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766305.848511:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098915200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766305.848523:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766305.848526:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766305.848528:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098915200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766305.848534:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098915328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766305.848549:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766305.848556:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766305/real 1775766305] req@ffff8b23864fe680 x1862022098915328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766305.848566:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098915328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766305.848572:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766305.848574:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766305.848576:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098915328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766305.848581:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098915456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766305.848593:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766305.848600:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766305/real 1775766305] req@ffff8b23864fca80 x1862022098915456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766305.848608:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098915456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766305.848614:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766305.848616:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766305.848619:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098915456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766306.743618:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766307.767753:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766308.791692:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766309.815738:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766310.839740:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766310.967631:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766310.967663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766310.967668:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766310.967676:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3772 00000100:00080000:0.0:1775766310.967686:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766310.967691:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766310.967727:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098915584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766310.967756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766310.967758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766310.967760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766310.967762:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3772 00000100:00080000:0.0:1775766310.967765:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766310.967767:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766310.967775:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766310.967784:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098915712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766310.967794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766310.967797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766310.967799:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766310.967801:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3772 00000100:00080000:0.0:1775766310.967804:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766310.967807:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766310.967811:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766310.967822:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098915840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766310.967830:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098915584:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766310.967833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766310.967839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766310.967842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766310.967847:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3772 00000100:00080000:0.0:1775766310.967849:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766310.967852:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766310.967855:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766310.967865:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098915968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766310.967871:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766310.967874:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766310.967875:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766310.967877:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3772 00000100:00080000:0.0:1775766310.967879:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766310.967881:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766310.967884:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766310.967894:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098916096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766310.967943:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766310.968012:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766310/real 1775766310] req@ffff8b2382dadf80 x1862022098915584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766310.968036:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098915584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766310.968048:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766310.968054:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766310.968058:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098915584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766310.968068:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098915712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766310.968095:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766310.968109:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766310/real 1775766310] req@ffff8b2382daf480 x1862022098915712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766310.968123:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098915712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766310.968131:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766310.968135:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766310.968140:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098915712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766310.968178:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098915840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766310.968201:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766310.968212:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766310/real 1775766310] req@ffff8b2382dace00 x1862022098915840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766310.968225:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098915840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766310.968234:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766310.968237:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766310.968241:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098915840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766310.968248:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098915968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766310.968269:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766310.968279:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766310/real 1775766310] req@ffff8b2382dafb80 x1862022098915968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766310.968293:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098915968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766310.968301:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766310.968305:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766310.968309:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098915968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766310.968315:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098916096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766310.968337:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766310.968349:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766310/real 1775766310] req@ffff8b2382dac700 x1862022098916096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766310.968362:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098916096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766310.968371:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766310.968375:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766310.968379:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098916096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766311.863684:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766312.887758:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766313.911789:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766314.935658:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766315.959815:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766316.087668:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766316.087679:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766316.087685:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766316.087695:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3777 00000100:00080000:3.0:1775766316.087709:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766316.087716:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766316.087766:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098916224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766316.087803:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766316.087807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766316.087810:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766316.087814:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3777 00000100:00080000:3.0:1775766316.087817:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766316.087821:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766316.087832:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766316.087845:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098916352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766316.087862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766316.087868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766316.087872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766316.087880:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098916224:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766316.087885:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3777 00000100:00080000:3.0:1775766316.087895:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766316.087902:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766316.087913:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766316.087937:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098916480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766316.087957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766316.087961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766316.087964:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766316.087969:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3777 00000100:00080000:3.0:1775766316.087974:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766316.087978:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766316.087985:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766316.088001:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098916608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766316.088012:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766316.088013:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098916352:192.168.202.128@tcp:38:kworker.0 00000100:00080000:3.0:1775766316.088018:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766316.088024:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766316.088031:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3777 00000100:00080000:3.0:1775766316.088038:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766316.088043:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766316.088048:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766316.088064:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098916736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766316.088075:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098916480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766316.088125:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098916608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766316.088158:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098916736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766316.983755:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766318.007698:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766319.031664:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775766319.159769:0:2365:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775766319.159808:0:2365:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775766319.159822:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766319.159858:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766319.159865:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766319.159871:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766319.159877:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775766319.160005:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766316/real 1775766319] req@ffff8b23864f3100 x1862022098916736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766319.160055:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098916736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766319.160074:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766319.160082:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766319.160089:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098916736:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766319.160101:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766316/real 1775766319] req@ffff8b23864f0000 x1862022098916480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766319.160128:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098916480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766319.160141:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766319.160147:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766319.160152:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098916480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766319.160162:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766316/real 1775766319] req@ffff8b23864f3b80 x1862022098916608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766319.160192:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098916608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766319.160203:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766319.160208:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766319.160212:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098916608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766319.160244:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766316/real 1775766319] req@ffff8b23864f1180 x1862022098916224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766319.160264:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098916224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766319.160275:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766319.160281:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766319.160287:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098916224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766319.160298:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766316/real 1775766319] req@ffff8b23864f0700 x1862022098916352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766319.160316:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098916352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766319.160327:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766319.160332:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766319.160338:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098916352:192.168.202.128@tcp:38:kworker.0 00010000:00010000:1.0:1775766320.055712:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766321.079629:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766321.207657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766321.207671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766321.207677:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766321.207691:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3782 00000100:00080000:0.0:1775766321.207708:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766321.207717:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766321.207772:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098916864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766321.207821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766321.207826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766321.207828:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766321.207832:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3782 00000100:00080000:0.0:1775766321.207836:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766321.207839:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766321.207852:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766321.207864:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098916992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766321.207877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766321.207882:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766321.207895:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766321.207903:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3782 00000100:00080000:0.0:1775766321.207910:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766321.207917:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766321.207925:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766321.207932:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098916864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766321.207948:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098917120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766321.207969:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766321.207976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766321.207979:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766321.207988:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3782 00000100:00080000:0.0:1775766321.207995:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766321.208002:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766321.208012:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766321.208032:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098917248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766321.208048:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766321.208054:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766321.208057:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766321.208063:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3782 00000100:00080000:0.0:1775766321.208071:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766321.208078:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766321.208085:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766321.208097:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766321.208106:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098917376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766321.208183:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766321/real 1775766321] req@ffff8b2382dac700 x1862022098916864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766321.208234:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098916864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766321.208252:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766321.208258:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766321.208265:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098916864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766321.208397:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098916992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766321.208445:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766321.208522:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766321/real 1775766321] req@ffff8b2382dafb80 x1862022098916992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766321.208561:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098916992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766321.208577:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766321.208583:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766321.208588:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098916992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766321.208634:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098917120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766321.208674:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766321.208703:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766321/real 1775766321] req@ffff8b2382dace00 x1862022098917120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766321.208724:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098917120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766321.208737:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766321.208742:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766321.208746:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098917120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766321.208755:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098917248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766321.208790:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766321.208806:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766321/real 1775766321] req@ffff8b2382daf480 x1862022098917248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766321.208823:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098917248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766321.208835:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766321.208840:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766321.208845:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098917248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766321.208855:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098917376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766321.208886:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766321.208903:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766321/real 1775766321] req@ffff8b2382dadf80 x1862022098917376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766321.208922:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098917376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766321.208934:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766321.208939:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766321.208944:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098917376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766322.103787:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766323.127682:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766324.151764:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766325.175797:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766326.199668:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766326.327623:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766326.327634:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766326.327639:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766326.327650:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3787 00000100:00080000:1.0:1775766326.327664:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766326.327671:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766326.327719:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098917504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766326.327747:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766326.327751:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766326.327753:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766326.327756:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3787 00000100:00080000:1.0:1775766326.327758:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766326.327761:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766326.327771:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766326.327780:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098917632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766326.327790:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766326.327793:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766326.327795:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766326.327798:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3787 00000100:00080000:1.0:1775766326.327800:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766326.327804:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766326.327808:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766326.327817:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098917760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766326.327823:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766326.327826:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766326.327828:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766326.327831:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3787 00000100:00080000:1.0:1775766326.327834:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766326.327838:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766326.327842:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766326.327848:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098917888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766326.327857:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766326.327860:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766326.327861:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766326.327864:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098917504:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766326.327865:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3787 00000100:00080000:1.0:1775766326.327869:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766326.327879:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766326.327889:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766326.327903:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098918016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766326.327981:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766326.328048:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766326/real 1775766326] req@ffff8b23864ff800 x1862022098917504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766326.328075:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098917504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766326.328092:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766326.328099:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766326.328105:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098917504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766326.328117:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098917632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766326.328162:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766326.328180:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766326/real 1775766326] req@ffff8b23864fc380 x1862022098917632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766326.328197:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098917632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766326.328208:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766326.328229:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766326.328234:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098917632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766326.328250:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098917760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766326.328280:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766326.328295:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766326/real 1775766326] req@ffff8b23864fed80 x1862022098917760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766326.328311:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098917760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766326.328323:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766326.328327:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766326.328332:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098917760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766326.328367:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098917888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766326.328397:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766326.328413:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766326/real 1775766326] req@ffff8b23864fc000 x1862022098917888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766326.328521:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098917888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766326.328544:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766326.328549:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766326.328554:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098917888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766326.328563:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098918016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766326.328594:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766326.328609:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766326/real 1775766326] req@ffff8b23864fdf80 x1862022098918016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766326.328625:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098918016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766326.328636:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766326.328641:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766326.328645:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098918016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766327.223627:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766328.247620:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766329.271668:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766330.295677:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766331.319682:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766331.447565:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766331.447576:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766331.447582:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766331.447594:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3792 00000100:00080000:1.0:1775766331.447609:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766331.447618:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766331.447668:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098918144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766331.447696:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766331.447699:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766331.447700:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766331.447704:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3792 00000100:00080000:1.0:1775766331.447707:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766331.447710:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766331.447737:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766331.447751:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098918272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766331.447761:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766331.447766:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766331.447767:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766331.447771:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3792 00000100:00080000:1.0:1775766331.447774:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766331.447775:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098918144:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766331.447778:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766331.447785:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766331.447794:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098918400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766331.447802:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766331.447821:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766331.447822:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766331.447826:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3792 00000100:00080000:1.0:1775766331.447829:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766331.447832:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766331.447836:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766331.447844:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098918528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766331.447850:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766331.447853:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766331.447854:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766331.447857:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3792 00000100:00080000:1.0:1775766331.447860:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766331.447862:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766331.447865:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766331.447871:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766331.447874:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098918656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766331.447932:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766331/real 1775766331] req@ffff8b23864fdf80 x1862022098918144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766331.447954:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098918144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766331.447962:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766331.447966:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766331.447970:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098918144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766331.448004:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098918272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766331.448021:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766331.448029:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766331/real 1775766331] req@ffff8b23864fc000 x1862022098918272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766331.448037:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098918272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766331.448043:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766331.448045:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766331.448047:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098918272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766331.448051:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098918400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766331.448062:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766331.448068:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766331/real 1775766331] req@ffff8b23864fed80 x1862022098918400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766331.448075:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098918400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766331.448085:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766331.448087:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766331.448089:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098918400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766331.448092:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098918528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766331.448102:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766331.448107:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766331/real 1775766331] req@ffff8b23864fc380 x1862022098918528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766331.448114:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098918528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766331.448119:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766331.448122:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766331.448124:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098918528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766331.448127:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098918656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766331.448137:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766331.448142:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766331/real 1775766331] req@ffff8b23864ff800 x1862022098918656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766331.448148:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098918656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766331.448153:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766331.448155:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766331.448157:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098918656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766332.343622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766333.367748:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766334.391603:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766335.415614:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766336.439625:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766336.567630:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766336.567640:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766336.567643:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766336.567652:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3797 00000100:00080000:0.0:1775766336.567663:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766336.567668:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766336.567706:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098918784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766336.567733:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766336.567736:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766336.567737:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766336.567740:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3797 00000100:00080000:0.0:1775766336.567742:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766336.567745:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766336.567753:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766336.567761:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098918912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766336.567770:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766336.567772:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766336.567775:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766336.567778:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3797 00000100:00080000:0.0:1775766336.567780:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766336.567783:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766336.567787:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766336.567804:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098919040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766336.567806:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098918784:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766336.567816:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766336.567819:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766336.567820:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766336.567824:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3797 00000100:00080000:0.0:1775766336.567825:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766336.567827:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766336.567831:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766336.567842:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098919168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766336.567848:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766336.567850:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766336.567851:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766336.567854:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3797 00000100:00080000:0.0:1775766336.567857:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766336.567860:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766336.567863:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766336.567873:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098919296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766336.567914:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766336.567973:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766336/real 1775766336] req@ffff8b2382daea00 x1862022098918784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766336.567993:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098918784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766336.568004:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766336.568009:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766336.568013:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098918784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766336.568029:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098918912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766336.568050:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766336.568060:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766336/real 1775766336] req@ffff8b2382dad500 x1862022098918912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766336.568070:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098918912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766336.568076:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766336.568079:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766336.568082:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098918912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766336.568111:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098919040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766336.568127:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766336.568134:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766336/real 1775766336] req@ffff8b2382dae680 x1862022098919040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766336.568143:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098919040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766336.568150:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766336.568152:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766336.568154:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098919040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766336.568160:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098919168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766336.568173:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766336.568179:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766336/real 1775766336] req@ffff8b2382dac380 x1862022098919168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766336.568187:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098919168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766336.568193:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766336.568196:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766336.568198:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098919168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766336.568202:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098919296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766336.568233:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766336.568241:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766336/real 1775766336] req@ffff8b2382daed80 x1862022098919296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766336.568258:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098919296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766336.568277:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766336.568280:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766336.568283:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098919296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766337.463631:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766338.487606:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766339.511707:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766340.535675:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766341.559615:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766341.687543:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766341.687550:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766341.687554:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766341.687562:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3803 00000100:00080000:3.0:1775766341.687572:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766341.687577:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766341.687612:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098919424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766341.687636:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766341.687639:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766341.687641:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766341.687643:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3803 00000100:00080000:3.0:1775766341.687646:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766341.687649:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766341.687657:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766341.687666:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098919552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766341.687670:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098919424:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766341.687682:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766341.687684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766341.687685:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766341.687688:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3803 00000100:00080000:3.0:1775766341.687690:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766341.687693:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766341.687696:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766341.687704:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098919680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766341.687710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766341.687712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766341.687713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766341.687715:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3803 00000100:00080000:3.0:1775766341.687718:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766341.687719:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766341.687723:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766341.687730:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098919808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766341.687736:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766341.687745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766341.687749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766341.687751:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766341.687757:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3803 00000100:00080000:3.0:1775766341.687761:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766341.687766:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766341.687772:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766341.687775:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766341/real 1775766341] req@ffff8b23864f2a00 x1862022098919424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766341.687788:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098919936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766341.687792:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098919424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766341.687824:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766341.687827:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766341.687830:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098919424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766341.687848:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098919552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766341.687862:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766341.687870:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766341/real 1775766341] req@ffff8b23864f2d80 x1862022098919552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766341.687878:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098919552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766341.687883:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766341.687885:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766341.687887:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098919552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766341.687890:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098919680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766341.687901:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766341.687906:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766341/real 1775766341] req@ffff8b23864f3480 x1862022098919680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766341.687913:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098919680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766341.687919:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766341.687921:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766341.687922:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098919680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766341.687926:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098919808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766341.687936:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766341.687942:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766341/real 1775766341] req@ffff8b23864f0700 x1862022098919808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766341.687949:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098919808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766341.687953:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766341.687955:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766341.687957:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098919808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766341.687960:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098919936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766341.687970:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766341.687975:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766341/real 1775766341] req@ffff8b23864f1180 x1862022098919936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766341.687981:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098919936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766341.687985:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766341.687997:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766341.687999:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098919936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766342.583688:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766343.607599:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766344.631687:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766345.655608:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766346.679628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766346.807571:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766346.807579:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766346.807583:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766346.807592:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3808 00000100:00080000:0.0:1775766346.807601:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766346.807606:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766346.807641:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098920064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766346.807667:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766346.807670:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766346.807671:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766346.807674:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3808 00000100:00080000:0.0:1775766346.807676:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766346.807679:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766346.807687:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766346.807701:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098920192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766346.807702:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098920064:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766346.807716:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766346.807719:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766346.807720:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766346.807723:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3808 00000100:00080000:0.0:1775766346.807726:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766346.807729:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766346.807731:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766346.807740:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098920320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766346.807750:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766346.807753:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766346.807754:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766346.807757:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3808 00000100:00080000:0.0:1775766346.807761:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766346.807763:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766346.807766:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766346.807775:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098920448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766346.807775:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766346.807783:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766346.807785:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766346.807787:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766346.807789:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3808 00000100:00080000:0.0:1775766346.807792:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766346.807794:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766346.807797:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766346.807807:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098920576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766346.807824:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766346/real 1775766346] req@ffff8b2382dac000 x1862022098920064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766346.807842:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098920064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766346.807851:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766346.807853:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766346.807856:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098920064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766346.807876:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098920192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766346.807892:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766346.807900:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766346/real 1775766346] req@ffff8b2382dad880 x1862022098920192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766346.807908:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098920192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766346.807918:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766346.807920:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766346.807922:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098920192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766346.807926:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098920320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766346.807937:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766346.807942:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766346/real 1775766346] req@ffff8b2382daca80 x1862022098920320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766346.807949:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098920320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766346.807954:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766346.807956:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766346.807958:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098920320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766346.807961:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098920448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766346.807972:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766346.807980:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766346/real 1775766346] req@ffff8b2382daf100 x1862022098920448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766346.807992:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098920448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766346.807999:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766346.808003:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766346.808006:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098920448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766346.808013:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098920576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766346.808033:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766346.808043:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766346/real 1775766346] req@ffff8b2382daed80 x1862022098920576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766346.808051:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098920576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766346.808057:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766346.808059:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766346.808061:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098920576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766347.703631:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766348.727622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766349.751585:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766350.775643:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766351.799611:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766351.927546:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766351.927554:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766351.927558:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766351.927571:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3813 00000100:00080000:3.0:1775766351.927581:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766351.927588:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766351.927624:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098920704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766351.927648:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766351.927651:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766351.927653:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766351.927656:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3813 00000100:00080000:3.0:1775766351.927658:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766351.927662:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766351.927670:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766351.927679:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098920832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766351.927684:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098920704:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766351.927694:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766351.927696:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766351.927697:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766351.927700:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3813 00000100:00080000:3.0:1775766351.927702:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766351.927704:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766351.927707:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766351.927716:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098920960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766351.927723:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766351.927725:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766351.927726:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766351.927729:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3813 00000100:00080000:3.0:1775766351.927732:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766351.927734:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766351.927737:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766351.927743:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766351.927747:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098921088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766351.927754:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766351.927757:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766351.927758:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766351.927761:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3813 00000100:00080000:3.0:1775766351.927763:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766351.927765:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766351.927768:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766351.927777:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098921216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766351.927781:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766351/real 1775766351] req@ffff8b23864f3b80 x1862022098920704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766351.927809:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098920704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766351.927817:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766351.927820:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766351.927823:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098920704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766351.927840:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098920832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766351.927855:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766351.927863:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766351/real 1775766351] req@ffff8b23864f0000 x1862022098920832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766351.927871:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098920832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766351.927876:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766351.927878:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766351.927881:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098920832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766351.927884:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098920960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766351.927895:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766351.927900:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766351/real 1775766351] req@ffff8b23864f3100 x1862022098920960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766351.927907:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098920960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766351.927917:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766351.927918:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766351.927920:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098920960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766351.927924:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098921088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766351.927942:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766351.927952:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766351/real 1775766351] req@ffff8b23864f1c00 x1862022098921088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766351.927963:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098921088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766351.927972:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766351.927975:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766351.927979:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098921088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766351.927985:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098921216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766351.928003:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766351.928010:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766351/real 1775766351] req@ffff8b23864f0380 x1862022098921216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766351.928017:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098921216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766351.928023:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766351.928025:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766351.928029:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098921216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766352.823635:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766353.847565:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766354.871733:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766355.895588:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766356.919734:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766357.047515:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766357.047526:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766357.047531:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766357.047541:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3818 00000100:00080000:3.0:1775766357.047552:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766357.047559:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766357.047598:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098921344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766357.047639:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766357.047643:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766357.047645:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766357.047650:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3818 00000100:00080000:3.0:1775766357.047653:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766357.047657:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766357.047667:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766357.047681:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098921472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766357.047694:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766357.047698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766357.047700:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766357.047705:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3818 00000100:00080000:3.0:1775766357.047709:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766357.047713:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766357.047719:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766357.047732:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098921600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766357.047746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766357.047750:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766357.047752:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766357.047759:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3818 00000100:00080000:3.0:1775766357.047765:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766357.047766:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098921344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766357.047773:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766357.047797:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766357.047812:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098921728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766357.047824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766357.047828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766357.047830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766357.047835:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3818 00000100:00080000:3.0:1775766357.047840:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766357.047845:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766357.047850:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766357.047862:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098921856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766357.047874:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766357.047945:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766357/real 1775766357] req@ffff8b23864f2680 x1862022098921344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766357.047974:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098921344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766357.047983:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766357.047987:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766357.047991:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098921344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766357.047998:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098921472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766357.048014:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766357.048022:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766357/real 1775766357] req@ffff8b23864f0a80 x1862022098921472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766357.048029:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098921472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766357.048046:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766357.048048:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766357.048049:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098921472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766357.048073:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098921600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766357.048084:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766357.048091:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766357/real 1775766357] req@ffff8b23864f1f80 x1862022098921600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766357.048099:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098921600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766357.048104:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766357.048106:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766357.048108:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098921600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766357.048112:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098921728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766357.048124:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766357.048129:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766357/real 1775766357] req@ffff8b23864f0380 x1862022098921728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766357.048137:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098921728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766357.048141:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766357.048143:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766357.048145:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098921728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766357.048149:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098921856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766357.048160:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766357.048166:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766357/real 1775766357] req@ffff8b23864f1c00 x1862022098921856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766357.048177:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098921856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766357.048186:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766357.048227:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766357.048232:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098921856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766357.943600:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766358.967629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766359.991636:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766361.015770:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766362.039712:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766362.167559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766362.167573:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766362.167579:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766362.167591:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3823 00000100:00080000:3.0:1775766362.167606:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766362.167614:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766362.167686:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098921984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766362.167715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766362.167718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766362.167720:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766362.167724:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3823 00000100:00080000:3.0:1775766362.167728:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766362.167731:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766362.167740:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766362.167752:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098922112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766362.167762:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766362.167765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766362.167766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766362.167770:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3823 00000100:00080000:3.0:1775766362.167772:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766362.167775:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766362.167778:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766362.167788:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098921984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766362.167789:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098922240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766362.167806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766362.167808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766362.167810:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766362.167813:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3823 00000100:00080000:3.0:1775766362.167814:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766362.167817:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766362.167820:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766362.167828:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098922368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766362.167836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766362.167838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766362.167840:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766362.167842:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3823 00000100:00080000:3.0:1775766362.167845:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766362.167847:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766362.167850:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766362.167857:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098922496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766362.167869:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766362.167909:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766362/real 1775766362] req@ffff8b23864f3100 x1862022098921984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766362.167925:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098921984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766362.167934:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766362.167937:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766362.167940:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098921984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766362.167945:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098922112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766362.167963:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766362.167971:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766362/real 1775766362] req@ffff8b23864f0000 x1862022098922112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766362.167980:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098922112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766362.167986:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766362.167989:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766362.167991:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098922112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766362.168011:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098922240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766362.168023:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766362.168030:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766362/real 1775766362] req@ffff8b23864f3b80 x1862022098922240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766362.168039:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098922240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766362.168044:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766362.168046:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766362.168048:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098922240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766362.168052:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098922368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766362.168073:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766362.168079:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766362/real 1775766362] req@ffff8b23864f1180 x1862022098922368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766362.168087:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098922368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766362.168092:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766362.168094:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766362.168096:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098922368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766362.168100:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098922496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766362.168112:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766362.168117:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766362/real 1775766362] req@ffff8b23864f0700 x1862022098922496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766362.168135:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098922496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766362.168141:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766362.168143:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766362.168145:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098922496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766363.063612:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766364.087722:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766365.111708:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766366.135665:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766367.159634:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766367.287559:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766367.287567:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766367.287570:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766367.287578:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3828 00000100:00080000:3.0:1775766367.287588:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766367.287593:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766367.287645:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098922624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766367.287671:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766367.287674:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766367.287676:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766367.287679:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3828 00000100:00080000:3.0:1775766367.287698:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766367.287700:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766367.287708:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766367.287718:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098922752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766367.287724:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766367.287727:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766367.287728:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766367.287731:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3828 00000100:00080000:3.0:1775766367.287733:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766367.287735:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766367.287738:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766367.287744:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098922880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766367.287750:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766367.287752:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766367.287754:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766367.287757:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3828 00000100:00080000:3.0:1775766367.287759:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766367.287762:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766367.287765:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766367.287768:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098922624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766367.287774:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098923008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766367.287793:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766367.287795:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766367.287797:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766367.287800:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3828 00000100:00080000:3.0:1775766367.287802:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766367.287805:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766367.287808:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766367.287816:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098923136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766367.287834:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766367.287872:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766367/real 1775766367] req@ffff8b23864f3480 x1862022098922624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766367.287886:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098922624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766367.287894:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766367.287897:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766367.287900:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098922624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766367.287905:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098922752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766367.287920:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766367.287927:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766367/real 1775766367] req@ffff8b23864f2d80 x1862022098922752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766367.287935:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098922752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766367.287944:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766367.287947:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766367.287949:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098922752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766367.287952:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098922880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766367.287963:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766367.287968:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766367/real 1775766367] req@ffff8b23864f2a00 x1862022098922880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766367.287975:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098922880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766367.287992:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766367.287994:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766367.287996:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098922880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766367.288026:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098923008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766367.288038:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766367.288044:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766367/real 1775766367] req@ffff8b23864f0700 x1862022098923008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766367.288051:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098923008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766367.288055:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766367.288057:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766367.288060:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098923008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766367.288063:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098923136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766367.288073:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766367.288078:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766367/real 1775766367] req@ffff8b23864f1180 x1862022098923136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766367.288085:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098923136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766367.288090:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766367.288091:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766367.288093:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098923136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766368.183626:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766369.207668:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766370.231601:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766371.255693:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766372.279738:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766372.407586:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766372.407615:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766372.407620:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766372.407634:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3833 00000100:00080000:3.0:1775766372.407650:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766372.407659:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766372.407709:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098923264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766372.407739:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766372.407742:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766372.407744:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766372.407751:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3833 00000100:00080000:3.0:1775766372.407771:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766372.407777:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766372.407790:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766372.407797:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098923264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766372.407810:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098923392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766372.407827:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766372.407831:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766372.407833:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766372.407839:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3833 00000100:00080000:3.0:1775766372.407843:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766372.407848:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766372.407853:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766372.407864:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098923520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766372.407874:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766372.407891:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766372.407895:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766372.407898:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766372.407905:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3833 00000100:00080000:3.0:1775766372.407909:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766372.407913:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766372.407918:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766372.407939:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098923648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766372.407941:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766372/real 1775766372] req@ffff8b23864f3b80 x1862022098923264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766372.407992:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766372.407996:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098923264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766372.408003:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766372.408009:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766372.408028:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3833 00000100:00080000:3.0:1775766372.408033:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766372.408034:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766372.408063:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00080000:3.0:1775766372.408066:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766372.408068:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098923264:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775766372.408098:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766372.408109:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098923392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766372.408120:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098923776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766372.408136:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766372.408162:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766372/real 1775766372] req@ffff8b23864f0000 x1862022098923392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766372.408178:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098923392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766372.408184:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766372.408188:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766372.408194:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098923392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766372.408199:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098923520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766372.408212:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766372.408218:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766372/real 1775766372] req@ffff8b23864f3100 x1862022098923520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766372.408225:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098923520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766372.408231:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766372.408232:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766372.408234:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098923520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766372.408247:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098923648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766372.408260:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766372.408266:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766372/real 1775766372] req@ffff8b23864f1c00 x1862022098923648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766372.408273:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098923648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766372.408278:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766372.408280:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766372.408282:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098923648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766372.408293:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098923776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766372.408303:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766372.408308:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766372/real 1775766372] req@ffff8b23864f0380 x1862022098923776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766372.408315:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098923776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766372.408320:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766372.408322:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766372.408324:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098923776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766373.303608:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766374.327690:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766375.351839:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766376.375776:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766377.399765:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766377.527582:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766377.527602:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766377.527611:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766377.527645:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3838 00000100:00080000:3.0:1775766377.527666:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766377.527679:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766377.527777:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098923904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766377.527835:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766377.527852:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766377.527855:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766377.527863:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3838 00000100:00080000:3.0:1775766377.527871:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766377.527878:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766377.527896:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766377.527927:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098924032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766377.527964:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766377.527977:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098923904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766377.527977:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766377.527981:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766377.528030:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3838 00000100:00080000:3.0:1775766377.528064:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766377.528072:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766377.528083:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766377.528129:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098924160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766377.528150:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766377.528156:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766377.528159:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766377.528166:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3838 00000800:00000100:0.0:1775766377.528177:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766377.528180:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766377.528192:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766377.528243:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766377.528268:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098924288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766377.528287:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766377/real 1775766377] req@ffff8b23864f1f80 x1862022098923904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766377.528323:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766377.528345:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098923904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766377.528347:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766377.528355:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766377.528373:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3838 00000100:00080000:0.0:1775766377.528374:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766377.528389:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766377.528390:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766377.528400:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098923904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766377.528407:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766377.528420:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766377.528443:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098924032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766377.528450:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098924416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766377.528483:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766377.528582:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766377/real 1775766377] req@ffff8b23864f0a80 x1862022098924032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766377.528601:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098924032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766377.528612:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766377.528618:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766377.528623:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098924032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766377.528630:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098924160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766377.528656:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766377.528668:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766377/real 1775766377] req@ffff8b23864f2680 x1862022098924160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766377.528681:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098924160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766377.528690:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766377.528693:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766377.528697:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098924160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766377.528704:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098924288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766377.528733:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766377.528743:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766377/real 1775766377] req@ffff8b23864f0380 x1862022098924288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766377.528757:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098924288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766377.528765:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766377.528769:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766377.528772:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098924288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766377.528791:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098924416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766377.528808:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766377.528818:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766377/real 1775766377] req@ffff8b23864f1c00 x1862022098924416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766377.528831:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098924416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766377.528840:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766377.528843:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766377.528846:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098924416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766378.423587:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766379.447786:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766380.471738:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766381.495572:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766382.519672:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766382.647702:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766382.647717:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766382.647724:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766382.647740:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3844 00000100:00080000:1.0:1775766382.647757:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766382.647766:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766382.647826:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098924544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766382.647877:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766382.647882:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766382.647885:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766382.647890:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3844 00000100:00080000:1.0:1775766382.647893:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766382.647898:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766382.647912:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766382.647929:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098924672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766382.647978:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766382.647984:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766382.647986:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766382.647993:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3844 00000100:00080000:1.0:1775766382.647998:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766382.648003:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766382.648009:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766382.648024:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098924800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766382.648037:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766382.648045:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1775766382.648048:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098924544:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766382.648048:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766382.648058:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3844 00000100:00080000:1.0:1775766382.648068:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766382.648072:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766382.648077:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766382.648090:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098924928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766382.648101:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766382.648105:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766382.648108:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766382.648113:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3844 00000100:00080000:1.0:1775766382.648117:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766382.648122:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766382.648127:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766382.648145:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098925056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766382.648260:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098924672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766382.648325:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098924800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766382.648412:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098924928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766382.648517:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098925056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766383.543734:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766384.567642:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766385.591673:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775766385.719836:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775766385.719889:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766385.719912:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766385.720025:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766385.720043:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766385.720060:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766385.720071:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775766385.720165:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766382/real 1775766385] req@ffff8b23864fce00 x1862022098924800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766385.720218:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098924800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766385.720238:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766385.720248:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766385.720254:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098924800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766385.720265:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766382/real 1775766385] req@ffff8b23864fd180 x1862022098924928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766385.720285:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098924928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766385.720298:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766385.720304:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766385.720310:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098924928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766385.720321:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766382/real 1775766385] req@ffff8b23864fea00 x1862022098925056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766385.720341:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098925056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766385.720353:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766385.720359:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766385.720364:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098925056:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766385.720376:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766382/real 1775766385] req@ffff8b23864fca80 x1862022098924544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766385.720395:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098924544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766385.720408:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766385.720414:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766385.720419:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098924544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766385.720430:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766382/real 1775766385] req@ffff8b23864fe680 x1862022098924672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766385.720513:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098924672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766385.720530:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766385.720547:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766385.720554:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098924672:192.168.202.128@tcp:38:kworker.0 00010000:00010000:3.0:1775766386.615732:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766387.639709:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766387.767613:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766387.767626:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766387.767631:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766387.767655:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3849 00000100:00080000:3.0:1775766387.767667:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766387.767674:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766387.767715:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098925184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766387.767748:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766387.767753:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766387.767757:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766387.767763:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3849 00000100:00080000:3.0:1775766387.767768:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766387.767784:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766387.767797:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766387.767816:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098925312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766387.767823:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098925184:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766387.767834:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766387.767848:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766387.767850:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766387.767856:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3849 00000100:00080000:3.0:1775766387.767860:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766387.767865:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766387.767871:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766387.767885:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098925440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766387.767897:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766387.767901:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766387.767925:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775766387.767930:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766387.767943:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3849 00000100:00080000:3.0:1775766387.767948:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766387.767952:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766387.767960:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766387.767975:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098925568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766387.767987:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766387/real 1775766387] req@ffff8b23864f3100 x1862022098925184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766387.767988:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766387.767993:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766387.767996:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766387.768001:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3849 00000100:00080000:3.0:1775766387.768013:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766387.768018:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766387.768021:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098925184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775766387.768025:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775766387.768037:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1775766387.768039:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098925696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766387.768044:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766387.768050:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098925184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766387.768080:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098925312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766387.768115:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766387.768142:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766387/real 1775766387] req@ffff8b23864f0000 x1862022098925312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766387.768159:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098925312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766387.768171:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766387.768176:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766387.768181:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098925312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766387.768190:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098925440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766387.768223:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766387.768240:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766387/real 1775766387] req@ffff8b23864f3b80 x1862022098925440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766387.768261:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098925440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766387.768274:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766387.768279:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766387.768284:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098925440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766387.768294:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098925568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766387.768327:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766387.768343:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766387/real 1775766387] req@ffff8b23864f1180 x1862022098925568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766387.768361:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098925568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766387.768374:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766387.768379:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766387.768383:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098925568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766387.768393:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098925696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766387.768435:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766387.768516:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766387/real 1775766387] req@ffff8b23864f0700 x1862022098925696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766387.768549:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098925696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766387.768562:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766387.768568:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766387.768573:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098925696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766388.663660:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766389.687713:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766390.711717:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766391.735592:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766392.759629:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766392.887604:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766392.887613:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766392.887618:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766392.887627:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3854 00000100:00080000:0.0:1775766392.887638:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766392.887644:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766392.887683:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098925824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766392.887715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766392.887719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766392.887721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766392.887724:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3854 00000100:00080000:0.0:1775766392.887727:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766392.887730:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766392.887740:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766392.887751:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098925952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766392.887756:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098925824:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766392.887773:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766392.887775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766392.887777:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766392.887781:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3854 00000100:00080000:0.0:1775766392.887784:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766392.887787:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766392.887791:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766392.887804:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098926080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766392.887823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766392.887826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766392.887829:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775766392.887834:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766392.887835:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3854 00000100:00080000:0.0:1775766392.887843:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766392.887846:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766392.887851:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766392.887881:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098926208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766392.887889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766392.887893:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766392.887894:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766392.887899:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766392/real 1775766392] req@ffff8b2382dac380 x1862022098925824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766392.887902:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3854 00000100:00080000:0.0:1775766392.887910:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766392.887917:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766392.887923:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766392.887932:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098925824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766392.887939:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098926336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766392.887956:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766392.887962:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766392.887967:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098925824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766392.887994:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098925952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766392.888015:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766392.888026:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766392/real 1775766392] req@ffff8b2382dae680 x1862022098925952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766392.888036:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098925952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766392.888043:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766392.888046:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766392.888049:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098925952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766392.888054:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098926080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766392.888067:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766392.888100:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766392/real 1775766392] req@ffff8b2382dad500 x1862022098926080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766392.888112:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098926080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766392.888120:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766392.888124:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766392.888129:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098926080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766392.888135:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098926208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766392.888153:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766392.888182:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766392/real 1775766392] req@ffff8b2382daea00 x1862022098926208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766392.888198:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098926208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766392.888207:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766392.888211:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766392.888214:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098926208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766392.888221:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098926336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766392.888267:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766392.888274:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766392/real 1775766392] req@ffff8b2382dadf80 x1862022098926336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766392.888284:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098926336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766392.888290:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766392.888292:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766392.888295:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098926336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766393.783660:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766394.807726:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766395.831739:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766396.855743:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766397.879631:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766398.007646:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766398.007657:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766398.007662:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766398.007673:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3859 00000100:00080000:3.0:1775766398.007688:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766398.007695:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766398.007744:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098926464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766398.007793:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766398.007850:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766398.007854:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766398.007862:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3859 00000100:00100000:0.0:1775766398.007870:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098926464:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766398.007872:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766398.007887:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766398.007902:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766398.007929:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098926592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766398.007951:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775766398.007955:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766398.007962:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766398.007965:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766398.007971:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3859 00000100:00080000:3.0:1775766398.007979:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766398.007986:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766398.007996:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766398.008007:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766398/real 1775766398] req@ffff8b23864f2a00 x1862022098926464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766398.008021:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098926720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766398.008033:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098926464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766398.008053:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766398.008063:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766398.008071:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098926464:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766398.008074:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766398.008079:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766398.008082:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766398.008089:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3859 00000100:00080000:3.0:1775766398.008096:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766398.008100:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098926592:192.168.202.128@tcp:38:kworker.0 00000100:00080000:3.0:1775766398.008105:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766398.008116:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766398.008135:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766398.008148:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098926848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766398.008152:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766398/real 1775766398] req@ffff8b23864f2d80 x1862022098926592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766398.008186:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766398.008191:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1775766398.008192:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098926592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766398.008197:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766398.008208:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3859 00000100:00080000:0.0:1775766398.008214:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766398.008216:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766398.008223:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00080000:3.0:1775766398.008226:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766398.008232:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098926592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766398.008245:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098926720:192.168.202.128@tcp:38:kworker.0 00000008:00100000:3.0:1775766398.008245:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766398.008288:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098926976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766398.008344:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766398.008361:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766398/real 1775766398] req@ffff8b23864f3480 x1862022098926720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766398.008376:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098926720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766398.008386:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766398.008390:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766398.008394:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098926720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766398.008411:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098926848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766398.008434:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766398.008444:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766398/real 1775766398] req@ffff8b23864f2a00 x1862022098926848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766398.008474:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098926848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766398.008509:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766398.008513:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766398.008516:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098926848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766398.008523:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098926976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766398.008545:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766398.008556:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766398/real 1775766398] req@ffff8b23864f0700 x1862022098926976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766398.008569:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098926976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766398.008577:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766398.008580:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766398.008583:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098926976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766398.903698:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766399.927722:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766400.951694:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766401.975665:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766402.999815:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766403.127547:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766403.127558:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766403.127565:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766403.127577:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3864 00000100:00080000:3.0:1775766403.127591:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766403.127598:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766403.127657:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098927104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766403.127687:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766403.127691:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766403.127692:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766403.127695:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3864 00000100:00080000:3.0:1775766403.127698:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766403.127701:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766403.127712:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766403.127724:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098927232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766403.127733:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766403.127736:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766403.127738:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766403.127741:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3864 00000100:00080000:3.0:1775766403.127743:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766403.127746:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766403.127791:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766403.127803:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098927360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766403.127812:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766403.127816:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766403.127818:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766403.127823:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3864 00000100:00080000:3.0:1775766403.127826:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766403.127829:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766403.127844:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766403.127852:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098927488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766403.127861:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766403.127865:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766403.127866:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766403.127870:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3864 00000100:00080000:3.0:1775766403.127873:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766403.127877:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766403.127884:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766403.127892:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098927104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766403.127893:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098927616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766403.127999:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766403.128050:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766403/real 1775766403] req@ffff8b23864f1180 x1862022098927104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766403.128079:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098927104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766403.128091:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766403.128096:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766403.128101:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098927104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766403.128109:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098927232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766403.128132:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766403.128143:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766403/real 1775766403] req@ffff8b23864f3b80 x1862022098927232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766403.128155:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098927232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766403.128162:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766403.128166:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766403.128169:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098927232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766403.128174:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098927360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766403.128193:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766403.128203:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766403/real 1775766403] req@ffff8b23864f0000 x1862022098927360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766403.128221:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098927360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766403.128244:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766403.128249:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766403.128254:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098927360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766403.128292:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098927488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766403.128326:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766403.128340:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766403/real 1775766403] req@ffff8b23864f3100 x1862022098927488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766403.128352:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098927488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766403.128360:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766403.128364:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766403.128367:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098927488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766403.128373:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098927616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766403.128392:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766403.128400:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766403/real 1775766403] req@ffff8b23864f1c00 x1862022098927616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766403.128411:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098927616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766403.128419:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766403.128421:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766403.128425:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098927616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766404.023664:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766405.047636:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766406.071707:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766407.095739:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766408.119584:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766408.247576:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766408.247585:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766408.247589:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766408.247599:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3869 00000100:00080000:3.0:1775766408.247612:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766408.247620:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766408.247682:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098927744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766408.247730:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766408.247735:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766408.247737:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766408.247740:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3869 00000100:00080000:3.0:1775766408.247744:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766408.247748:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766408.247757:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766408.247771:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098927872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766408.247782:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766408.247786:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766408.247787:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766408.247791:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3869 00000100:00080000:3.0:1775766408.247793:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766408.247800:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766408.247801:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098927744:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775766408.247804:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766408.247818:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098928000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766408.247825:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766408.247828:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766408.247830:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766408.247833:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3869 00000100:00080000:3.0:1775766408.247837:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766408.247840:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766408.247844:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766408.247854:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098928128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766408.247861:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766408.247864:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766408.247866:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766408.247868:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3869 00000100:00080000:3.0:1775766408.247871:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766408.247873:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766408.247876:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766408.247885:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098928256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766408.247895:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766408.247946:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766408/real 1775766408] req@ffff8b23864f0380 x1862022098927744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766408.247964:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098927744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766408.247974:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766408.247978:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766408.247981:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098927744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766408.247988:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098927872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766408.248010:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766408.248020:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766408/real 1775766408] req@ffff8b23864f2680 x1862022098927872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766408.248030:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098927872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766408.248036:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766408.248039:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766408.248043:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098927872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766408.248064:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098928000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766408.248077:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766408.248085:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766408/real 1775766408] req@ffff8b23864f0a80 x1862022098928000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766408.248095:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098928000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766408.248101:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766408.248104:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766408.248106:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098928000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766408.248111:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098928128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766408.248124:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766408.248131:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766408/real 1775766408] req@ffff8b23864f1f80 x1862022098928128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766408.248140:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098928128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766408.248146:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766408.248148:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766408.248150:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098928128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766408.248154:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098928256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766408.248167:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766408.248174:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766408/real 1775766408] req@ffff8b23864f1c00 x1862022098928256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766408.248182:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098928256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766408.248188:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766408.248191:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766408.248193:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098928256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766409.143619:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766410.167750:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766411.191622:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766412.215594:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766413.239717:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766413.367623:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766413.367671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766413.367676:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766413.367698:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3874 00000100:00080000:1.0:1775766413.367711:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766413.367718:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766413.367765:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098928384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766413.367794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766413.367798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766413.367800:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766413.367804:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3874 00000100:00080000:1.0:1775766413.367807:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766413.367811:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766413.367821:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766413.367848:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098928512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766413.367860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766413.367864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766413.367866:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766413.367874:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3874 00000100:00100000:0.0:1775766413.367877:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098928384:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766413.367878:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766413.367886:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766413.367894:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766413.367908:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098928640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766413.367920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766413.367924:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766413.367926:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766413.367941:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3874 00000100:00080000:1.0:1775766413.367954:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766413.367958:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766413.367963:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766413.367966:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766413.367978:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098928768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766413.367990:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766413.367993:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766413.367994:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766413.367998:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3874 00000100:00080000:1.0:1775766413.368002:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766413.368006:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766413.368011:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766413.368016:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766413/real 1775766413] req@ffff8b23864fdc00 x1862022098928384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766413.368025:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098928896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766413.368065:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098928384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766413.368079:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766413.368084:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766413.368089:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098928384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766413.368096:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098928512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766413.368120:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766413.368146:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766413/real 1775766413] req@ffff8b23864fd500 x1862022098928512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766413.368166:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098928512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766413.368179:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766413.368184:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766413.368190:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098928512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766413.368241:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098928640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766413.368274:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766413.368292:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766413/real 1775766413] req@ffff8b23864ff100 x1862022098928640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766413.368311:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098928640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766413.368320:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766413.368323:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766413.368326:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098928640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766413.368332:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098928768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766413.368356:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766413.368366:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766413/real 1775766413] req@ffff8b23864fe300 x1862022098928768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766413.368376:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098928768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766413.368383:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766413.368386:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766413.368389:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098928768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766413.368403:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098928896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766413.368418:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766413.368438:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766413/real 1775766413] req@ffff8b23864ffb80 x1862022098928896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766413.368494:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098928896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766413.368521:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766413.368526:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766413.368532:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098928896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766414.263695:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766415.287718:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766416.311690:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766417.335775:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766418.359807:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766418.487632:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766418.487640:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766418.487644:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766418.487652:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3879 00000100:00080000:3.0:1775766418.487663:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766418.487668:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766418.487702:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098929024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766418.487734:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766418.487737:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766418.487739:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766418.487741:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3879 00000100:00080000:3.0:1775766418.487744:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766418.487746:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766418.487755:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766418.487765:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098929152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766418.487774:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766418.487777:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766418.487778:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766418.487782:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3879 00000100:00080000:3.0:1775766418.487785:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766418.487786:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098929024:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766418.487790:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766418.487798:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766418.487804:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098929280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766418.487812:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766418.487814:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766418.487816:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766418.487821:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3879 00000100:00080000:3.0:1775766418.487825:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766418.487829:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766418.487834:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766418.487849:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098929408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766418.487850:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766418.487862:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766418.487866:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766418.487868:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766418.487873:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3879 00000100:00080000:3.0:1775766418.487877:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766418.487882:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766418.487887:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766418/real 1775766418] req@ffff8b23864f3100 x1862022098929024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775766418.487888:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766418.487905:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098929536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766418.487909:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098929024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766418.487933:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766418.487938:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766418.487942:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098929024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766418.487947:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098929152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766418.487965:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766418.487972:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766418/real 1775766418] req@ffff8b23864f0000 x1862022098929152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766418.487981:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098929152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766418.488000:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766418.488002:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766418.488005:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098929152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766418.488026:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098929280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766418.488048:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766418.488059:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766418/real 1775766418] req@ffff8b23864f3b80 x1862022098929280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766418.488072:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098929280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766418.488081:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766418.488085:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766418.488087:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098929280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766418.488091:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098929408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766418.488107:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766418.488114:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766418/real 1775766418] req@ffff8b23864f1180 x1862022098929408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766418.488122:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098929408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766418.488127:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766418.488129:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766418.488131:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098929408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766418.488135:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098929536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766418.488147:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766418.488153:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766418/real 1775766418] req@ffff8b23864f0700 x1862022098929536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766418.488164:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098929536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766418.488172:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766418.488175:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766418.488179:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098929536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766419.383683:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766420.407659:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766421.431601:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766422.455599:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766423.479798:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766423.607700:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766423.607714:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766423.607722:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766423.607738:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3885 00000100:00080000:3.0:1775766423.607757:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766423.607768:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766423.607827:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098929664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766423.607864:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766423.607868:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766423.607870:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766423.607874:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3885 00000100:00080000:3.0:1775766423.607878:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766423.607881:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766423.607892:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766423.607906:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098929792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766423.607917:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766423.607921:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766423.607923:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766423.607930:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3885 00000100:00080000:3.0:1775766423.607933:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766423.607937:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766423.607941:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766423.607952:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098929920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766423.607962:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766423.607965:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766423.607968:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766423.607975:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3885 00000100:00100000:0.0:1775766423.607977:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098929664:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766423.607980:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766423.607990:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766423.607998:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766423.608015:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098930048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766423.608024:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766423.608029:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766423.608031:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766423.608036:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3885 00000100:00080000:3.0:1775766423.608041:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766423.608044:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766423.608049:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766423.608057:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098930176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766423.608069:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766423.608125:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766423/real 1775766423] req@ffff8b23864f2a00 x1862022098929664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766423.608145:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098929664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766423.608156:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766423.608160:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766423.608164:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098929664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766423.608172:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098929792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766423.608192:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766423.608204:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766423/real 1775766423] req@ffff8b23864f3480 x1862022098929792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766423.608215:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098929792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766423.608222:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766423.608224:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766423.608227:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098929792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766423.608262:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098929920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766423.608278:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766423.608286:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766423/real 1775766423] req@ffff8b23864f2d80 x1862022098929920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766423.608296:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098929920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766423.608303:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766423.608307:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766423.608309:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098929920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766423.608327:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098930048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766423.608344:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766423.608353:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766423/real 1775766423] req@ffff8b23864f0700 x1862022098930048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766423.608363:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098930048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766423.608370:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766423.608373:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766423.608376:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098930048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766423.608381:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098930176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766423.608396:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766423.608403:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766423/real 1775766423] req@ffff8b23864f1180 x1862022098930176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766423.608418:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098930176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766423.608430:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766423.608434:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766423.608530:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098930176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766424.503646:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766425.527631:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766426.551708:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766427.575604:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766428.599623:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766428.727568:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766428.727576:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766428.727580:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766428.727587:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3890 00000100:00080000:1.0:1775766428.727597:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766428.727603:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766428.727646:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098930304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766428.727676:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766428.727679:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766428.727681:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766428.727684:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3890 00000100:00080000:1.0:1775766428.727687:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766428.727690:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766428.727699:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766428.727710:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098930432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766428.727717:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098930304:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766428.727722:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766428.727725:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766428.727726:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766428.727730:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3890 00000100:00080000:1.0:1775766428.727732:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766428.727735:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766428.727738:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766428.727747:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098930560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766428.727758:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766428.727760:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766428.727761:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766428.727764:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3890 00000100:00080000:1.0:1775766428.727767:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766428.727769:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766428.727772:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766428.727775:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766428.727784:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098930688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766428.727791:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766428.727794:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766428.727795:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766428.727798:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3890 00000100:00080000:1.0:1775766428.727800:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766428.727803:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766428.727811:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766428/real 1775766428] req@ffff8b23864ffb80 x1862022098930304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775766428.727814:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766428.727830:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098930816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766428.727833:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098930304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766428.727856:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766428.727860:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766428.727863:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098930304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766428.727885:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098930432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766428.727900:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766428.727908:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766428/real 1775766428] req@ffff8b23864fe300 x1862022098930432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766428.727916:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098930432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766428.727921:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766428.727923:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766428.727925:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098930432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766428.727929:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098930560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766428.727939:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766428.727944:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766428/real 1775766428] req@ffff8b23864ff100 x1862022098930560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766428.727951:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098930560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766428.727956:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766428.727958:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766428.727960:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098930560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766428.727963:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098930688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766428.727974:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766428.727979:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766428/real 1775766428] req@ffff8b23864fd500 x1862022098930688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766428.727987:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098930688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766428.727991:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766428.727993:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766428.727995:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098930688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766428.727998:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098930816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766428.728008:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766428.728012:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766428/real 1775766428] req@ffff8b23864fdc00 x1862022098930816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766428.728020:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098930816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766428.728025:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766428.728026:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766428.728028:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098930816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766429.623615:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766430.647768:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766431.671599:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766432.695802:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766433.719718:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766433.847673:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766433.847716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766433.847723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766433.847739:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3895 00000100:00080000:0.0:1775766433.847756:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766433.847766:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766433.847822:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098930944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766433.847868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766433.847873:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766433.847875:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766433.847880:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3895 00000100:00080000:0.0:1775766433.847884:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766433.847888:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766433.847900:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766433.847917:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098931072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766433.847932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766433.847937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766433.847940:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766433.847946:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3895 00000100:00080000:0.0:1775766433.847949:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766433.847953:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766433.847959:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766433.847972:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098931200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766433.847988:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766433.847989:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098930944:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766433.847994:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766433.848000:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766433.848028:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3895 00000100:00080000:0.0:1775766433.848032:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766433.848037:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766433.848043:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766433.848066:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098931328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766433.848079:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766433.848083:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766433.848085:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766433.848089:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3895 00000100:00080000:0.0:1775766433.848094:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766433.848099:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766433.848106:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766433.848126:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766433.848132:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098931456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766433.848197:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766433/real 1775766433] req@ffff8b2382daf480 x1862022098930944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766433.848234:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098930944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766433.848248:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766433.848254:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766433.848259:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098930944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766433.848270:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098931072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766433.848298:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766433.848312:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766433/real 1775766433] req@ffff8b2382dace00 x1862022098931072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766433.848326:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098931072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766433.848336:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766433.848340:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766433.848344:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098931072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766433.848381:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098931200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766433.848401:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766433.848412:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766433/real 1775766433] req@ffff8b2382dafb80 x1862022098931200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766433.848426:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098931200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766433.848557:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766433.848562:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766433.848566:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098931200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766433.848575:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098931328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766433.848607:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766433.848621:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766433/real 1775766433] req@ffff8b2382dac700 x1862022098931328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766433.848636:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098931328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766433.848646:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766433.848650:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766433.848654:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098931328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766433.848660:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098931456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766433.848678:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766433.848705:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766433/real 1775766433] req@ffff8b2382dadf80 x1862022098931456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766433.848718:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098931456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766433.848727:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766433.848730:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766433.848734:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098931456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766434.743628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766435.767659:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766436.791594:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766437.815619:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766438.839656:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766438.967552:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766438.967561:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766438.967565:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766438.967573:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3900 00000100:00080000:1.0:1775766438.967582:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766438.967587:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766438.967622:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098931584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766438.967646:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766438.967648:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766438.967650:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766438.967652:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3900 00000100:00080000:1.0:1775766438.967654:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766438.967657:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766438.967665:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766438.967674:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098931712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766438.967692:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766438.967693:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098931584:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766438.967695:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766438.967698:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766438.967704:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3900 00000100:00080000:1.0:1775766438.967706:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766438.967709:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766438.967712:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766438.967721:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098931840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766438.967726:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766438.967729:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766438.967730:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766438.967733:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3900 00000100:00080000:1.0:1775766438.967735:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766438.967737:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766438.967740:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766438.967746:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098931968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766438.967752:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766438.967755:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766438.967757:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766438.967761:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3900 00000100:00080000:1.0:1775766438.967765:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775766438.967769:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775766438.967772:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766438.967778:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766438.967806:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098932096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766438.967819:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766438/real 1775766438] req@ffff8b23864fe680 x1862022098931584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766438.967835:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098931584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766438.967843:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766438.967847:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766438.967850:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098931584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766438.967869:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098931712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766438.967885:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766438.967893:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766438/real 1775766438] req@ffff8b23864fca80 x1862022098931712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766438.967901:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098931712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766438.967906:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766438.967908:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766438.967911:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098931712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766438.967914:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098931840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766438.967925:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766438.967931:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766438/real 1775766438] req@ffff8b23864fea00 x1862022098931840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766438.967938:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098931840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766438.967943:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766438.967944:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766438.967946:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098931840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766438.967949:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098931968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766438.967959:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766438.967964:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766438/real 1775766438] req@ffff8b23864fd180 x1862022098931968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766438.967971:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098931968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766438.967976:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766438.967978:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766438.967980:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098931968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766438.967983:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098932096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766438.967992:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766438.967997:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766438/real 1775766438] req@ffff8b23864fce00 x1862022098932096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766438.968009:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098932096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766438.968013:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766438.968015:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766438.968017:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098932096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766439.863737:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766440.887643:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766441.911668:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766442.935583:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766443.959700:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766444.087564:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766444.087572:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766444.087576:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766444.087584:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3905 00000100:00080000:3.0:1775766444.087594:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766444.087599:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766444.087634:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098932224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766444.087661:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766444.087665:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766444.087667:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766444.087671:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3905 00000100:00080000:3.0:1775766444.087674:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766444.087679:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766444.087688:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766444.087701:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098932352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766444.087713:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766444.087714:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098932224:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766444.087716:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766444.087718:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766444.087722:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3905 00000100:00080000:3.0:1775766444.087725:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766444.087729:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766444.087733:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766444.087743:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098932480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766444.087751:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766444.087754:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766444.087755:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766444.087759:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3905 00000100:00080000:3.0:1775766444.087762:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766444.087765:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766444.087769:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766444.087774:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766444.087777:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098932608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766444.087784:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766444.087787:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766444.087789:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766444.087792:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3905 00000100:00080000:3.0:1775766444.087795:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766444.087798:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766444.087802:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766444.087810:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098932736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766444.087813:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766444/real 1775766444] req@ffff8b23864f3b80 x1862022098932224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766444.087830:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098932224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766444.087841:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766444.087845:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766444.087849:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098932224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766444.087871:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098932352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766444.087889:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766444.087899:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766444/real 1775766444] req@ffff8b23864f0000 x1862022098932352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766444.087907:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098932352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766444.087912:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766444.087915:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766444.087917:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098932352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766444.087921:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098932480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766444.087931:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766444.087937:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766444/real 1775766444] req@ffff8b23864f3100 x1862022098932480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766444.087944:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098932480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766444.087949:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766444.087950:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766444.087952:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098932480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766444.087955:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098932608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766444.087968:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766444.087973:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766444/real 1775766444] req@ffff8b23864f1c00 x1862022098932608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766444.087980:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098932608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766444.087985:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766444.087987:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766444.087989:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098932608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766444.087992:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098932736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766444.088001:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766444.088006:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766444/real 1775766444] req@ffff8b23864f1f80 x1862022098932736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766444.088013:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098932736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766444.088018:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766444.088020:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766444.088029:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098932736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766444.983597:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766446.007615:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766447.031606:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766448.055657:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766449.079636:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766449.207618:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766449.207630:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766449.207636:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766449.207647:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3910 00000100:00080000:1.0:1775766449.207661:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766449.207668:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766449.207714:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098932864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766449.207760:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766449.207764:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766449.207766:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766449.207769:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3910 00000100:00080000:1.0:1775766449.207772:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766449.207775:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766449.207785:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766449.207796:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098932992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766449.207806:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766449.207809:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766449.207811:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766449.207815:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3910 00000100:00080000:1.0:1775766449.207818:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766449.207821:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766449.207826:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766449.207834:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098933120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766449.207842:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766449.207845:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766449.207849:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766449.207851:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098932864:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766449.207855:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3910 00000100:00080000:1.0:1775766449.207861:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766449.207864:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766449.207883:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766449.207893:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098933248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766449.207900:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766449.207903:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766449.207904:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766449.207907:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3910 00000100:00080000:1.0:1775766449.207910:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766449.207912:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766449.207915:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766449.207927:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098933376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766449.207999:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098932992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766449.208037:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098933120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766449.208054:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098933248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766449.208071:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098933376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766450.103641:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766451.127609:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766452.151622:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775766452.279720:0:2363:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:3.0:1775766452.279784:0:2363:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775766452.279835:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775766452.279876:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775766452.279886:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775766452.279929:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775766452.279940:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775766452.279950:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766449/real 1775766452] req@ffff8b23864fed80 x1862022098933120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766452.279982:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098933120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766452.279996:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766452.280004:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766452.280008:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098933120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766452.280020:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766449/real 1775766452] req@ffff8b23864fc000 x1862022098933248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766452.280043:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098933248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766452.280052:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766452.280056:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766452.280061:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098933248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766452.280069:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766449/real 1775766452] req@ffff8b23864fdf80 x1862022098933376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766452.280083:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098933376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766452.280092:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766452.280096:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766452.280100:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098933376:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766452.280108:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766449/real 1775766452] req@ffff8b23864ff800 x1862022098932864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766452.280123:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098932864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766452.280142:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766452.280146:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766452.280151:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098932864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766452.280158:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766449/real 1775766452] req@ffff8b23864fc380 x1862022098932992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766452.280174:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098932992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766452.280184:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766452.280187:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766452.280224:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098932992:192.168.202.128@tcp:38:kworker.0 00010000:00010000:1.0:1775766453.175580:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766454.199641:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766454.327601:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766454.327611:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766454.327615:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766454.327625:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3915 00000100:00080000:0.0:1775766454.327636:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766454.327644:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766454.327681:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098933504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766454.327716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766454.327721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766454.327724:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766454.327729:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3915 00000100:00080000:0.0:1775766454.327733:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766454.327737:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766454.327765:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766454.327782:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098933632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766454.327793:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098933504:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766454.327801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766454.327808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766454.327811:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766454.327832:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3915 00000100:00080000:0.0:1775766454.327838:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766454.327844:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766454.327849:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766454.327865:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098933760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766454.327881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1775766454.327881:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766454.327889:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766454.327891:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766454.327899:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3915 00000100:00080000:0.0:1775766454.327907:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766454.327912:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766454.327918:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766454.327934:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766454/real 1775766454] req@ffff8b2382daea00 x1862022098933504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766454.327946:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098933888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766454.327962:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766454.327968:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098933504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766454.327969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766454.327974:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766454.327985:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3915 00000100:00080000:0.0:1775766454.327994:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766454.327996:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766454.328001:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:3.0:1775766454.328008:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000008:00100000:0.0:1775766454.328009:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766454.328017:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098933504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766454.328027:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098934016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766454.328054:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098933632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766454.328089:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766454.328105:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766454/real 1775766454] req@ffff8b2382dad500 x1862022098933632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766454.328121:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098933632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766454.328131:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766454.328135:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766454.328139:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098933632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766454.328147:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098933760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766454.328181:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766454.328196:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766454/real 1775766454] req@ffff8b2382dae680 x1862022098933760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766454.328212:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098933760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766454.328235:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766454.328239:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766454.328244:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098933760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766454.328251:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098933888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766454.328277:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766454.328291:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766454/real 1775766454] req@ffff8b2382dac380 x1862022098933888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766454.328305:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098933888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766454.328314:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766454.328567:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766454.328573:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098933888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766454.328581:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098934016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766454.328612:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766454.328626:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766454/real 1775766454] req@ffff8b2382daed80 x1862022098934016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766454.328640:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098934016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766454.328650:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766454.328654:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766454.328659:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098934016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766455.223747:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766456.247769:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766457.271711:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766458.295679:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766459.319637:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766459.447527:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766459.447534:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766459.447538:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766459.447546:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3920 00000100:00080000:0.0:1775766459.447556:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766459.447563:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766459.447610:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098934144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766459.447639:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766459.447643:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766459.447645:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766459.447648:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3920 00000100:00080000:0.0:1775766459.447652:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766459.447655:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766459.447663:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766459.447673:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098934272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766459.447682:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766459.447684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766459.447685:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766459.447689:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3920 00000100:00080000:0.0:1775766459.447691:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766459.447694:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766459.447697:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766459.447705:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098934400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766459.447713:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098934144:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766459.447714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766459.447719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766459.447720:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766459.447725:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3920 00000100:00080000:0.0:1775766459.447731:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766459.447735:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766459.447740:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766459.447754:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098934528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766459.447765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766459.447769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766459.447771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766459.447775:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3920 00000100:00080000:0.0:1775766459.447779:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775766459.447784:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766459.447785:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766459.447791:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766459.447802:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098934656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766459.447825:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766459/real 1775766459] req@ffff8b2382daf100 x1862022098934144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766459.447841:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098934144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766459.447849:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766459.447852:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766459.447855:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098934144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766459.447860:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098934272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766459.447881:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766459.447890:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766459/real 1775766459] req@ffff8b2382daca80 x1862022098934272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766459.447898:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098934272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766459.447903:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766459.447905:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766459.447908:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098934272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766459.447928:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098934400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766459.447939:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766459.447945:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766459/real 1775766459] req@ffff8b2382dad880 x1862022098934400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766459.447952:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098934400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766459.447962:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766459.447964:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766459.447966:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098934400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766459.447970:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098934528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766459.447980:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766459.447985:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766459/real 1775766459] req@ffff8b2382dac000 x1862022098934528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766459.447992:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098934528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766459.447997:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766459.447999:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766459.448000:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098934528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766459.448004:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098934656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766459.448012:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766459.448017:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766459/real 1775766459] req@ffff8b2382daed80 x1862022098934656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766459.448023:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098934656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766459.448028:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766459.448030:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766459.448033:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098934656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766460.343660:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766461.367718:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766462.391582:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766463.415655:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766464.439578:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766464.567646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766464.567657:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766464.567662:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766464.567694:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3925 00000100:00080000:0.0:1775766464.567705:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766464.567713:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766464.567757:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098934784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766464.567792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766464.567797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766464.567799:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766464.567803:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3925 00000100:00080000:0.0:1775766464.567809:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766464.567813:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766464.567824:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766464.567824:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098934784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766464.567845:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098934912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766464.567858:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766464.567862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766464.567864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766464.567869:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3925 00000100:00080000:0.0:1775766464.567874:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766464.567879:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766464.567884:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766464.567889:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766464.567913:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098935040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766464.567925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766464.567927:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766464/real 1775766464] req@ffff8b2382dac380 x1862022098934784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766464.567932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766464.567937:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766464.567946:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3925 00000100:00100000:3.0:1775766464.567950:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098934784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766464.567951:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766464.567961:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:3.0:1775766464.567964:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1775766464.567968:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775766464.567969:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766464.567975:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098934784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766464.567988:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098935168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766464.567999:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098934912:192.168.202.128@tcp:38:kworker.0 00000100:00080000:0.0:1775766464.568011:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766464.568015:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766464.568018:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766464.568024:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3925 00000800:00000100:3.0:1775766464.568025:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766464.568032:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766464.568038:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766464.568039:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766464/real 1775766464] req@ffff8b2382dae680 x1862022098934912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775766464.568048:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766464.568051:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098934912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766464.568059:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766464.568062:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766464.568064:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098934912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766464.568070:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098935040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766464.568075:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098935296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766464.568084:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766464.568094:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766464/real 1775766464] req@ffff8b2382dad500 x1862022098935040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766464.568103:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098935040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766464.568123:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766464.568125:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766464.568127:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098935040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766464.568138:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098935168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766464.568153:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766464.568159:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766464/real 1775766464] req@ffff8b2382daea00 x1862022098935168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766464.568167:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098935168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766464.568172:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766464.568174:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766464.568176:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098935168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766464.568179:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098935296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766464.568188:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766464.568193:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766464/real 1775766464] req@ffff8b2382dadf80 x1862022098935296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766464.568201:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098935296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766464.568205:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766464.568207:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766464.568209:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098935296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766465.463677:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766466.487576:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766467.511647:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766468.535570:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766469.559645:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766469.687575:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766469.687583:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766469.687587:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766469.687595:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3931 00000100:00080000:3.0:1775766469.687606:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766469.687611:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766469.687646:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098935424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766469.687682:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766469.687685:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766469.687686:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766469.687689:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3931 00000100:00080000:3.0:1775766469.687691:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766469.687694:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766469.687702:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766469.687714:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098935424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766469.687714:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098935552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766469.687728:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766469.687730:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766469.687732:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766469.687735:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3931 00000100:00080000:3.0:1775766469.687737:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766469.687740:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766469.687742:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766469.687750:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098935680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766469.687756:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766469.687759:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766469.687760:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766469.687763:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3931 00000100:00080000:3.0:1775766469.687765:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766469.687768:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766469.687770:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766469.687778:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098935808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766469.687785:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775766469.687786:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766469.687790:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766469.687792:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766469.687795:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3931 00000100:00080000:3.0:1775766469.687798:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766469.687800:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766469.687803:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766469.687809:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098935936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766469.687832:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766469/real 1775766469] req@ffff8b23864f0a80 x1862022098935424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766469.687847:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098935424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766469.687855:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766469.687858:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766469.687861:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098935424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766469.687878:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098935552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766469.687894:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766469.687902:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766469/real 1775766469] req@ffff8b23864f2680 x1862022098935552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766469.687923:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098935552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766469.687931:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766469.687935:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766469.687938:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098935552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766469.687944:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098935680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766469.687964:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766469.687974:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766469/real 1775766469] req@ffff8b23864f0380 x1862022098935680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766469.687995:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098935680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766469.688001:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766469.688003:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766469.688005:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098935680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766469.688009:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098935808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766469.688025:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766469.688031:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766469/real 1775766469] req@ffff8b23864f1f80 x1862022098935808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766469.688039:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098935808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766469.688048:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766469.688051:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766469.688066:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098935808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766469.688071:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098935936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766469.688084:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766469.688091:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766469/real 1775766469] req@ffff8b23864f1c00 x1862022098935936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766469.688098:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098935936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766469.688103:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766469.688105:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766469.688107:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098935936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766470.583642:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766471.607731:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766472.631627:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766473.655638:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766474.679739:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766474.807542:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766474.807552:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766474.807558:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766474.807568:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3936 00000100:00080000:3.0:1775766474.807580:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766474.807587:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766474.807651:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098936064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766474.807692:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766474.807697:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766474.807700:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766474.807704:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3936 00000100:00080000:3.0:1775766474.807708:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766474.807712:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766474.807723:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766474.807740:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098936192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766474.807754:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766474.807758:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766474.807761:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766474.807766:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3936 00000100:00080000:3.0:1775766474.807770:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766474.807775:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766474.807780:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766474.807796:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098936064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766474.807802:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098936320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766474.807823:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766474.807829:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766474.807832:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766474.807840:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3936 00000100:00080000:3.0:1775766474.807848:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766474.807855:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766474.807864:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766474.807884:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098936448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766474.807898:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766474.807910:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766474.807915:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766474.807917:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766474.807941:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3936 00000100:00080000:3.0:1775766474.807949:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766474.807954:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766474.807962:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766474.807985:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766474/real 1775766474] req@ffff8b23864f3100 x1862022098936064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766474.807986:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098936576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766474.808062:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098936064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766474.808072:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766474.808076:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766474.808080:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098936064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766474.808087:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098936192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766474.808130:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766474.808146:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766474/real 1775766474] req@ffff8b23864f0000 x1862022098936192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766474.808161:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098936192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766474.808184:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766474.808187:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766474.808189:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098936192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766474.808210:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098936320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766474.808222:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766474.808229:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766474/real 1775766474] req@ffff8b23864f3b80 x1862022098936320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766474.808236:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098936320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766474.808259:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766474.808262:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766474.808265:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098936320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766474.808272:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098936448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766474.808291:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766474.808301:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766474/real 1775766474] req@ffff8b23864f1180 x1862022098936448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766474.808315:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098936448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766474.808323:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766474.808326:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766474.808329:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098936448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766474.808334:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098936576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766474.808352:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766474.808360:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766474/real 1775766474] req@ffff8b23864f0700 x1862022098936576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766474.808374:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098936576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766474.808399:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766474.808403:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766474.808406:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098936576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766475.703750:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766476.727580:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766477.751639:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766478.775739:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766479.799623:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766479.927503:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766479.927520:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766479.927524:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766479.927535:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3941 00000100:00080000:3.0:1775766479.927546:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766479.927553:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766479.927590:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098936704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766479.927615:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766479.927618:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766479.927619:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766479.927622:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3941 00000100:00080000:3.0:1775766479.927625:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766479.927627:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766479.927635:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766479.927645:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098936832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766479.927655:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766479.927658:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766479.927659:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766479.927663:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3941 00000100:00080000:3.0:1775766479.927665:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766479.927667:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766479.927670:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766479.927675:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098936704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766479.927680:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098936960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766479.927690:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766479.927692:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766479.927693:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766479.927696:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3941 00000100:00080000:3.0:1775766479.927698:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766479.927700:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766479.927703:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766479.927711:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098937088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766479.927717:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766479.927719:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766479.927720:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766479.927723:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3941 00000100:00080000:3.0:1775766479.927725:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766479.927728:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766479.927733:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766479.927739:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766479.927746:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098937216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766479.927774:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766479/real 1775766479] req@ffff8b23864f2d80 x1862022098936704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766479.927788:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098936704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766479.927800:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766479.927803:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766479.927806:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098936704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766479.927811:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098936832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766479.927825:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766479.927832:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766479/real 1775766479] req@ffff8b23864f3480 x1862022098936832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766479.927840:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098936832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766479.927845:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766479.927846:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766479.927848:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098936832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766479.927865:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098936960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766479.927875:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766479.927881:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766479/real 1775766479] req@ffff8b23864f2a00 x1862022098936960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766479.927888:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098936960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766479.927894:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766479.927896:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766479.927898:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098936960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766479.927901:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098937088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766479.927923:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766479.927928:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766479/real 1775766479] req@ffff8b23864f0700 x1862022098937088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766479.927935:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098937088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766479.927940:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766479.927942:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766479.927944:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098937088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766479.927960:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098937216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766479.927975:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766479.927982:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766479/real 1775766479] req@ffff8b23864f1180 x1862022098937216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766479.927988:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098937216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766479.928004:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766479.928006:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766479.928008:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098937216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766480.823884:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766481.847826:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766482.871747:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766483.895675:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766484.919571:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766485.047560:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766485.047569:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766485.047573:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766485.047581:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3946 00000100:00080000:0.0:1775766485.047591:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766485.047596:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766485.047632:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098937344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766485.047657:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766485.047660:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766485.047661:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766485.047664:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3946 00000100:00080000:0.0:1775766485.047666:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766485.047668:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766485.047676:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766485.047685:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098937472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766485.047691:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098937344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766485.047696:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766485.047699:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766485.047701:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766485.047704:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3946 00000100:00080000:0.0:1775766485.047706:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766485.047709:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766485.047713:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766485.047723:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098937600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766485.047733:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766485.047735:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766485.047737:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766485.047740:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3946 00000100:00080000:0.0:1775766485.047742:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766485.047744:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766485.047747:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766485.047751:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766485.047756:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098937728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766485.047773:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766485.047777:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766485.047779:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766485.047799:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3946 00000100:00080000:0.0:1775766485.047804:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766485.047805:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766485/real 1775766485] req@ffff8b2382dac700 x1862022098937344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766485.047813:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766485.047818:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766485.047840:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098937856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766485.047843:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098937344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766485.047866:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766485.047886:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766485.047889:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098937344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766485.047922:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098937472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766485.047946:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766485.047960:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766485/real 1775766485] req@ffff8b2382dafb80 x1862022098937472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766485.047972:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098937472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766485.047986:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766485.047989:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766485.047992:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098937472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766485.047999:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098937600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766485.048020:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766485.048028:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766485/real 1775766485] req@ffff8b2382dace00 x1862022098937600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766485.048036:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098937600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766485.048042:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766485.048044:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766485.048046:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098937600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766485.048050:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098937728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766485.048062:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766485.048067:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766485/real 1775766485] req@ffff8b2382daf480 x1862022098937728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766485.048075:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098937728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766485.048079:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766485.048081:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766485.048084:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098937728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766485.048087:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098937856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766485.048096:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766485.048101:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766485/real 1775766485] req@ffff8b2382dadf80 x1862022098937856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766485.048108:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098937856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766485.048113:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766485.048115:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766485.048116:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098937856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766485.943600:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766486.967615:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766487.991602:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766489.015741:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766490.039662:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766490.167604:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766490.167625:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766490.167631:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766490.167644:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3951 00000100:00080000:0.0:1775766490.167660:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766490.167668:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766490.167720:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098937984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766490.167756:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766490.167761:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766490.167764:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766490.167786:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3951 00000100:00080000:0.0:1775766490.167791:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766490.167797:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766490.167814:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766490.167837:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098938112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766490.167874:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766490.167904:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766490.167907:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766490.167919:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3951 00000100:00100000:3.0:1775766490.167924:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098937984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766490.167926:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766490.167937:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766490.167952:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766490.167975:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098938240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766490.167990:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766490.167995:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766490.167999:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766490.168005:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3951 00000100:00080000:0.0:1775766490.168010:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766490.168016:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766490.168022:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766490.168038:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098938368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766490.168051:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1775766490.168061:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766490.168068:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766490.168073:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766490.168082:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3951 00000100:00080000:0.0:1775766490.168092:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766490.168097:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766490.168104:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766490.168131:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098938496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766490.168149:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766490/real 1775766490] req@ffff8b2382daea00 x1862022098937984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766490.168180:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098937984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766490.168195:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766490.168201:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766490.168206:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098937984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766490.168218:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098938112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766490.168250:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766490.168266:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766490/real 1775766490] req@ffff8b2382dad500 x1862022098938112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766490.168295:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098938112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766490.168306:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766490.168311:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766490.168315:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098938112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766490.168351:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098938240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766490.168374:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766490.168387:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766490/real 1775766490] req@ffff8b2382dae680 x1862022098938240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766490.168401:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098938240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766490.168412:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766490.168416:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766490.168420:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098938240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766490.168427:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098938368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766490.168446:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766490.168526:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766490/real 1775766490] req@ffff8b2382dac380 x1862022098938368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766490.168564:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098938368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766490.168580:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766490.168586:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766490.168593:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098938368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766490.168606:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098938496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766490.168648:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766490.168668:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766490/real 1775766490] req@ffff8b2382daed80 x1862022098938496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766490.168683:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098938496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766490.168694:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766490.168698:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766490.168702:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098938496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766491.063698:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766492.087649:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766493.111633:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766494.135598:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766495.159661:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766495.287604:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766495.287617:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766495.287622:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766495.287637:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3956 00000100:00080000:3.0:1775766495.287650:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766495.287659:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766495.287706:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098938624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766495.287736:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766495.287747:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766495.287749:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766495.287752:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3956 00000100:00080000:3.0:1775766495.287756:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766495.287760:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766495.287770:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766495.287779:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098938752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766495.287806:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766495.287809:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766495.287811:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766495.287817:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3956 00000100:00100000:0.0:1775766495.287818:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098938624:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766495.287819:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766495.287828:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766495.287835:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766495.287851:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098938880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766495.287864:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766495.287868:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766495.287871:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766495.287876:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3956 00000100:00080000:3.0:1775766495.287881:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766495.287886:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766495.287892:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766495.287901:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766495.287905:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098939008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766495.287913:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766495.287916:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766495.287918:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766495.287922:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3956 00000100:00080000:3.0:1775766495.287925:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766495.287929:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766495.287933:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766495.287942:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098939136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766495.287964:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766495/real 1775766495] req@ffff8b23864f3b80 x1862022098938624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766495.287988:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098938624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766495.288004:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766495.288008:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766495.288013:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098938624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766495.288038:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098938752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766495.288058:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766495.288069:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766495/real 1775766495] req@ffff8b23864f0000 x1862022098938752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766495.288079:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098938752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766495.288085:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766495.288087:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766495.288090:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098938752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766495.288094:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098938880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766495.288108:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766495.288115:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766495/real 1775766495] req@ffff8b23864f3100 x1862022098938880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766495.288123:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098938880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766495.288129:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766495.288131:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766495.288133:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098938880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766495.288148:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098939008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766495.288163:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766495.288169:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766495/real 1775766495] req@ffff8b23864f1c00 x1862022098939008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766495.288177:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098939008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766495.288183:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766495.288185:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766495.288187:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098939008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766495.288207:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098939136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766495.288222:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766495.288229:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766495/real 1775766495] req@ffff8b23864f1f80 x1862022098939136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766495.288243:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098939136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766495.288264:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766495.288268:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766495.288272:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098939136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766496.183702:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766497.207631:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766498.231688:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766499.255670:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766500.279649:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766500.407601:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766500.407609:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766500.407613:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766500.407621:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3961 00000100:00080000:2.0:1775766500.407636:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766500.407642:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766500.407680:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022098939264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766500.407703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766500.407706:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766500.407707:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766500.407710:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3961 00000100:00080000:2.0:1775766500.407712:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766500.407714:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766500.407722:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766500.407765:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098939392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766500.407778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766500.407782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766500.407795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766500.407800:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3961 00000100:00080000:2.0:1775766500.407804:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766500.407809:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766500.407814:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766500.407824:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098939520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766500.407847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766500.407850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766500.407851:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766500.407855:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3961 00000100:00080000:2.0:1775766500.407859:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766500.407861:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766500.407865:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766500.407873:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022098939648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766500.407882:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766500.407884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766500.407887:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766500.407895:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3961 00000100:00100000:0.0:1775766500.407898:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098939264:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775766500.407899:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766500.407913:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766500.407918:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766500.407927:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098939776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766500.408019:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766500.408090:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766500/real 1775766500] req@ffff8b23b9ea7b80 x1862022098939264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766500.408112:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022098939264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766500.408124:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766500.408130:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766500.408134:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098939264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766500.408142:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098939392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766500.408164:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766500.408176:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766500/real 1775766500] req@ffff8b23b9ea5f80 x1862022098939392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766500.408188:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022098939392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766500.408195:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766500.408198:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766500.408201:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098939392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766500.408206:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098939520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766500.408224:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766500.408232:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766500/real 1775766500] req@ffff8b23b9ea6680 x1862022098939520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766500.408243:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022098939520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766500.408257:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766500.408260:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766500.408263:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098939520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766500.408296:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098939648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766500.408312:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766500.408320:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766500/real 1775766500] req@ffff8b23b9ea6d80 x1862022098939648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766500.408330:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022098939648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766500.408337:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766500.408340:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766500.408343:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098939648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766500.408349:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098939776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766500.408363:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766500.408371:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766500/real 1775766500] req@ffff8b23b9ea5c00 x1862022098939776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766500.408382:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022098939776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766500.408388:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766500.408391:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766500.408395:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098939776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766501.303688:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766502.327622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766503.351674:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766504.375725:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766505.399595:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766505.527544:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766505.527555:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766505.527561:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766505.527572:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3966 00000100:00080000:0.0:1775766505.527585:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766505.527593:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766505.527656:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098939904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766505.527714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766505.527725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766505.527727:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766505.527733:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3966 00000100:00080000:0.0:1775766505.527737:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766505.527744:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766505.527745:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098939904:192.168.202.128@tcp:250:kworker.0 00000008:00100000:0.0:1775766505.527755:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766505.527773:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098940032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766505.527787:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766505.527790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766505.527792:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766505.527798:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3966 00000100:00080000:0.0:1775766505.527802:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775766505.527806:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766505.527810:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766505.527816:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766505.527830:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098940160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766505.527841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766505.527841:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766505/real 1775766505] req@ffff8b2382dac000 x1862022098939904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766505.527850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766505.527854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766505.527862:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3966 00000100:00080000:0.0:1775766505.527870:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766505.527874:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766505.527879:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098939904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775766505.527882:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775766505.527892:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766505.527895:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766505.527901:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098940288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766505.527901:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098939904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766505.527916:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766505.527919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766505.527922:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766505.527925:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098940032:192.168.202.128@tcp:38:kworker.0 00000100:00080000:0.0:1775766505.527930:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3966 00000100:00080000:0.0:1775766505.527935:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766505.527940:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766505.527946:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766505.527946:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766505.527956:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766505/real 1775766505] req@ffff8b2382dad880 x1862022098940032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766505.527964:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098940416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766505.527969:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098940032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766505.527989:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766505.527993:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766505.527995:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098940032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766505.527999:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098940160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766505.528022:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766505.528029:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766505/real 1775766505] req@ffff8b2382daca80 x1862022098940160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766505.528036:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098940160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766505.528046:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766505.528048:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766505.528050:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098940160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766505.528053:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098940288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766505.528064:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766505.528069:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766505/real 1775766505] req@ffff8b2382daf100 x1862022098940288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766505.528076:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098940288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766505.528081:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766505.528083:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766505.528084:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098940288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766505.528097:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098940416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766505.528106:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766505.528111:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766505/real 1775766505] req@ffff8b2382dac000 x1862022098940416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766505.528118:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098940416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766505.528122:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766505.528124:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766505.528126:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098940416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766506.423760:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766507.447643:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766508.471741:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766509.495719:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766510.519658:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766510.647652:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766510.647664:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766510.647669:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766510.647683:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3972 00000100:00080000:3.0:1775766510.647698:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766510.647706:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766510.647757:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098940544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766510.647788:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766510.647791:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766510.647793:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766510.647796:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3972 00000100:00080000:3.0:1775766510.647799:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766510.647803:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766510.647813:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766510.647823:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098940672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766510.647833:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098940544:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766510.647833:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766510.647836:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766510.647837:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766510.647844:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3972 00000100:00080000:3.0:1775766510.647848:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766510.647851:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766510.647855:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766510.647864:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098940800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766510.647870:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766510.647873:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766510.647875:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766510.647878:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3972 00000100:00080000:3.0:1775766510.647881:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766510.647883:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766510.647886:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766510.647895:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098940928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766510.647900:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766510.647902:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766510.647904:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766510.647906:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3972 00000100:00080000:3.0:1775766510.647909:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766510.647912:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766510.647915:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766510.647919:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766510.647926:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098941056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766510.647975:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766510/real 1775766510] req@ffff8b23864f0380 x1862022098940544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766510.647999:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098940544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766510.648012:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766510.648015:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766510.648019:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098940544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766510.648042:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098940672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766510.648062:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766510.648071:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766510/real 1775766510] req@ffff8b23864f2680 x1862022098940672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766510.648080:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098940672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766510.648085:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766510.648088:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766510.648090:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098940672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766510.648093:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098940800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766510.648104:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766510.648110:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766510/real 1775766510] req@ffff8b23864f0a80 x1862022098940800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766510.648122:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098940800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766510.648127:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766510.648129:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766510.648131:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098940800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766510.648134:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098940928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766510.648145:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766510.648151:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766510/real 1775766510] req@ffff8b23864f1f80 x1862022098940928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766510.648158:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098940928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766510.648163:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766510.648165:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766510.648167:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098940928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766510.648170:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098941056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766510.648180:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766510.648185:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766510/real 1775766510] req@ffff8b23864f1c00 x1862022098941056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766510.648191:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098941056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766510.648200:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766510.648203:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766510.648204:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098941056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766511.543647:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766512.567679:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766513.591732:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766514.615607:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766515.639754:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766515.767614:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766515.767625:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766515.767631:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766515.767643:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3977 00000100:00080000:0.0:1775766515.767656:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766515.767663:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766515.767708:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098941184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766515.767744:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766515.767749:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766515.767752:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766515.767757:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3977 00000100:00080000:0.0:1775766515.767763:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766515.767769:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766515.767783:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766515.767789:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098941184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766515.767803:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098941312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766515.767815:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766515.767818:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766515.767820:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766515.767825:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3977 00000100:00080000:0.0:1775766515.767828:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766515.767833:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766515.767838:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766515.767848:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098941440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766515.767856:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766515.767859:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766515.767860:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766515.767863:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3977 00000100:00080000:0.0:1775766515.767875:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766515.767878:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766515.767881:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766515.767889:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098941568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766515.767897:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766515.767900:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766515.767901:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766515.767904:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3977 00000100:00080000:0.0:1775766515.767906:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766515.767909:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766515.767913:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766515.767929:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098941696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766515.767948:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098941312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766515.767986:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098941440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766515.768004:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098941568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766515.768020:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098941696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766516.663628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766517.687632:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766518.711693:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775766518.839823:0:2364:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775766518.839871:0:2364:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766518.839890:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766518.839935:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766518.839947:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766518.839958:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766518.839969:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:3.0:1775766518.840018:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766515/real 1775766518] req@ffff8b2382dac380 x1862022098941312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766518.840053:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098941312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766518.840075:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766518.840081:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766518.840086:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098941312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766518.840097:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766515/real 1775766518] req@ffff8b2382dae680 x1862022098941440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766518.840111:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098941440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766518.840131:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766518.840136:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766518.840140:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098941440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766518.840148:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766515/real 1775766518] req@ffff8b2382dad500 x1862022098941568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766518.840164:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098941568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766518.840173:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766518.840178:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766518.840182:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098941568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766518.840190:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766515/real 1775766518] req@ffff8b2382daea00 x1862022098941696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766518.840205:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098941696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766518.840214:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766518.840218:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766518.840222:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098941696:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766518.840231:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766515/real 1775766518] req@ffff8b2382daed80 x1862022098941184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766518.840246:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098941184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766518.840256:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766518.840261:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766518.840265:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098941184:192.168.202.128@tcp:250:kworker.0 00010000:00010000:3.0:1775766519.735681:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766520.759682:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766520.887666:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766520.887680:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766520.887686:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766520.887701:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3982 00000100:00080000:0.0:1775766520.887716:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766520.887724:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766520.887767:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098941824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766520.887803:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766520.887808:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766520.887810:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766520.887814:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3982 00000100:00080000:0.0:1775766520.887818:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766520.887831:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766520.887843:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766520.887858:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098941952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766520.887863:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098941824:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766520.887881:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766520.887885:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766520.887887:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766520.887893:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3982 00000100:00080000:0.0:1775766520.887896:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766520.887902:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766520.887907:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766520.887920:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098942080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766520.887930:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766520.887934:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766520.887936:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766520.887940:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3982 00000100:00080000:0.0:1775766520.887944:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766520.887949:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766520.887955:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766520.887966:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098942208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766520.887972:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766520.887985:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766520.887989:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766520.888005:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766520.888010:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3982 00000100:00080000:0.0:1775766520.888015:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766520.888020:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766520.888026:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766520.888036:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766520/real 1775766520] req@ffff8b2382dadf80 x1862022098941824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766520.888045:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098942336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766520.888069:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098941824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766520.888084:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766520.888089:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766520.888094:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098941824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766520.888132:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098941952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766520.888162:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766520.888175:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766520/real 1775766520] req@ffff8b2382daf480 x1862022098941952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766520.888188:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098941952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766520.888196:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766520.888200:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766520.888204:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098941952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766520.888210:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098942080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766520.888233:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766520.888246:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766520/real 1775766520] req@ffff8b2382dace00 x1862022098942080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766520.888260:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098942080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766520.888269:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766520.888273:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766520.888278:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098942080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766520.888287:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098942208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766520.888317:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766520.888331:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766520/real 1775766520] req@ffff8b2382dafb80 x1862022098942208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766520.888346:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098942208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766520.888354:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766520.888358:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766520.888361:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098942208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766520.888367:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098942336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766520.888570:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766520.888582:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766520/real 1775766520] req@ffff8b2382dac700 x1862022098942336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766520.888594:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098942336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766520.888604:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766520.888607:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766520.888611:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098942336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766521.783599:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766522.807604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766523.831650:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766524.855599:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766525.879640:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766526.007582:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766526.007593:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766526.007598:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766526.007607:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3987 00000100:00080000:2.0:1775766526.007620:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766526.007627:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766526.007670:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098942464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766526.007698:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766526.007702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766526.007704:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766526.007709:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3987 00000100:00080000:2.0:1775766526.007713:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766526.007717:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766526.007727:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766526.007733:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098942464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775766526.007743:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098942592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766526.007756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766526.007760:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766526.007762:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766526.007767:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3987 00000100:00080000:2.0:1775766526.007780:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766526.007784:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766526.007790:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766526.007796:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775766526.007805:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098942720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766526.007817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766526.007821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766526.007822:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766526.007827:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3987 00000100:00080000:2.0:1775766526.007831:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766526.007835:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766526.007841:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766526.007845:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766526/real 1775766526] req@ffff8b23b9ea5500 x1862022098942464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766526.007856:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098942848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766526.007869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766526.007869:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022098942464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766526.007877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766526.007882:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766526.007890:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3987 00000100:00080000:2.0:1775766526.007894:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766526.007898:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:3.0:1775766526.007900:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1775766526.007903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775766526.007908:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766526.007914:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098942464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775766526.007920:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022098942976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766526.007943:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098942592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766526.007971:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766526.007984:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766526/real 1775766526] req@ffff8b23b9ea4700 x1862022098942592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766526.008004:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022098942592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766526.008013:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766526.008016:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766526.008020:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098942592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766526.008027:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098942720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766526.008048:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766526.008057:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766526/real 1775766526] req@ffff8b23b9ea6a00 x1862022098942720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766526.008066:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022098942720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766526.008072:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766526.008074:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766526.008077:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098942720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766526.008081:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098942848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766526.008094:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766526.008100:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766526/real 1775766526] req@ffff8b23b9ea4000 x1862022098942848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766526.008107:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022098942848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766526.008117:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766526.008119:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766526.008121:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098942848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766526.008124:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098942976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766526.008134:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766526.008138:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766526/real 1775766526] req@ffff8b23b9ea5880 x1862022098942976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766526.008146:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022098942976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766526.008154:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766526.008158:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766526.008160:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098942976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766526.903639:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766527.927596:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766528.951603:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766529.975615:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766530.999605:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766531.127577:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766531.127585:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766531.127590:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766531.127598:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3992 00000100:00080000:0.0:1775766531.127608:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766531.127613:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766531.127648:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098943104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766531.127671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766531.127674:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766531.127676:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766531.127678:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3992 00000100:00080000:0.0:1775766531.127680:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766531.127682:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766531.127690:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766531.127701:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098943232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766531.127709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766531.127712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766531.127713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766531.127716:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3992 00000100:00080000:0.0:1775766531.127719:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766531.127723:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098943104:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766531.127724:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766531.127731:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766531.127739:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098943360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766531.127746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766531.127749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766531.127750:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766531.127753:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3992 00000100:00080000:0.0:1775766531.127755:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766531.127757:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766531.127760:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766531.127767:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098943488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766531.127773:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766531.127776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766531.127777:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766531.127779:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3992 00000100:00080000:0.0:1775766531.127782:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775766531.127785:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766531.127788:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766531.127792:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766531.127803:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098943616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766531.127825:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766531/real 1775766531] req@ffff8b2382dac700 x1862022098943104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766531.127849:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098943104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766531.127859:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766531.127865:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766531.127869:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098943104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766531.127877:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098943232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766531.127902:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766531.127916:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766531/real 1775766531] req@ffff8b2382dafb80 x1862022098943232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766531.127927:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098943232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766531.127933:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766531.127935:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766531.127938:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098943232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766531.127958:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098943360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766531.127982:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766531.127989:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766531/real 1775766531] req@ffff8b2382dace00 x1862022098943360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766531.127997:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098943360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766531.128003:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766531.128005:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766531.128007:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098943360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766531.128011:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098943488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766531.128023:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766531.128028:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766531/real 1775766531] req@ffff8b2382daf480 x1862022098943488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766531.128035:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098943488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766531.128040:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766531.128042:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766531.128044:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098943488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766531.128047:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098943616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766531.128063:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766531.128069:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766531/real 1775766531] req@ffff8b2382dadf80 x1862022098943616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766531.128075:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098943616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766531.128080:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766531.128083:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766531.128085:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098943616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766532.023660:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766533.047607:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766534.071577:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766535.095624:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766536.119581:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766536.247607:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766536.247618:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766536.247639:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766536.247653:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 3997 00000100:00080000:3.0:1775766536.247668:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766536.247684:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766536.247731:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098943744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766536.247760:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766536.247764:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766536.247765:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766536.247768:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 3997 00000100:00080000:3.0:1775766536.247771:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766536.247775:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766536.247785:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766536.247794:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098943872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766536.247805:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098943744:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766536.247808:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766536.247816:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766536.247818:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766536.247823:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3997 00000100:00080000:3.0:1775766536.247828:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766536.247832:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766536.247854:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766536.247868:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098944000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766536.247887:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766536.247890:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766536.247891:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766536.247895:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3997 00000100:00080000:3.0:1775766536.247899:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766536.247902:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766536.247907:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766536.247916:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766536.247918:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098944128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766536.247924:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766536.247927:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766536.247929:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766536.247932:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 3997 00000100:00080000:3.0:1775766536.247934:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766536.247936:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766536.247938:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766536.247944:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098944256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766536.247986:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766536/real 1775766536] req@ffff8b23864f3100 x1862022098943744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766536.248008:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098943744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766536.248017:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766536.248021:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766536.248024:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098943744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766536.248047:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098943872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766536.248066:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766536.248074:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766536/real 1775766536] req@ffff8b23864f0000 x1862022098943872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766536.248082:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098943872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766536.248088:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766536.248090:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766536.248092:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098943872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766536.248096:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098944000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766536.248107:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766536.248112:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766536/real 1775766536] req@ffff8b23864f3b80 x1862022098944000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766536.248119:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098944000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766536.248129:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766536.248131:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766536.248134:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098944000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766536.248137:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098944128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766536.248148:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766536.248153:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766536/real 1775766536] req@ffff8b23864f1180 x1862022098944128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766536.248160:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098944128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766536.248165:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766536.248167:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766536.248168:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098944128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766536.248172:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098944256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766536.248181:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766536.248186:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766536/real 1775766536] req@ffff8b23864f0700 x1862022098944256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766536.248192:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098944256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766536.248197:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766536.248200:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766536.248202:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098944256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766537.143706:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766538.167701:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766539.191654:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766540.215575:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766541.239771:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766541.367618:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766541.367634:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766541.367642:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766541.367657:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4002 00000100:00080000:2.0:1775766541.367672:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766541.367681:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766541.367732:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098944384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766541.367770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766541.367775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766541.367778:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766541.367782:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4002 00000100:00080000:2.0:1775766541.367787:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766541.367792:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766541.367804:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766541.367819:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098944512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766541.367835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766541.367839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766541.367841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766541.367847:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4002 00000100:00080000:2.0:1775766541.367851:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766541.367855:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766541.367861:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766541.367866:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098944384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775766541.367878:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098944640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766541.367914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766541.367919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766541.367922:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766541.367930:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4002 00000100:00080000:2.0:1775766541.367937:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766541.367945:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766541.367954:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766541.367975:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098944768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766541.367983:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775766541.368003:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766541.368010:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766541.368014:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766541.368023:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4002 00000100:00080000:2.0:1775766541.368031:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766541.368039:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766541.368049:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766541.368054:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766541/real 1775766541] req@ffff8b23b9ea4a80 x1862022098944384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766541.368073:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098944896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766541.368093:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022098944384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766541.368109:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766541.368116:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766541.368122:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098944384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766541.368131:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098944512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766541.368164:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766541.368180:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766541/real 1775766541] req@ffff8b23b9ea7100 x1862022098944512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766541.368195:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022098944512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766541.368204:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766541.368208:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766541.368212:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098944512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766541.368241:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098944640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766541.368262:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766541.368273:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766541/real 1775766541] req@ffff8b23b9ea4380 x1862022098944640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766541.368287:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022098944640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766541.368297:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766541.368300:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766541.368343:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098944640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766541.368352:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098944768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766541.368380:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766541.368393:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766541/real 1775766541] req@ffff8b23b9ea4e00 x1862022098944768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766541.368408:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022098944768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766541.368417:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766541.368421:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766541.368424:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098944768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766541.368431:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098944896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766541.368449:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766541.368531:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766541/real 1775766541] req@ffff8b23b9ea7480 x1862022098944896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766541.368552:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022098944896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766541.368563:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766541.368568:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766541.368572:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098944896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766542.263682:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766543.287615:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766544.311574:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766545.335651:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766546.359596:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766546.487581:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766546.487592:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766546.487597:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766546.487608:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4007 00000100:00080000:3.0:1775766546.487621:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766546.487628:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766546.487684:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098945024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766546.487712:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766546.487716:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766546.487718:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766546.487723:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4007 00000100:00080000:3.0:1775766546.487728:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766546.487733:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766546.487745:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766546.487763:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098945152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766546.487770:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098945024:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766546.487776:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766546.487780:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766546.487782:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766546.487786:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4007 00000100:00080000:3.0:1775766546.487789:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766546.487811:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766546.487822:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766546.487850:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098945280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766546.487870:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766546.487877:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766546.487885:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766546.487889:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766546.487916:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4007 00000100:00080000:3.0:1775766546.487925:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766546.487933:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766546.487944:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766546.487955:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766546/real 1775766546] req@ffff8b23864f2a00 x1862022098945024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766546.487985:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098945408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766546.488003:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766546.488009:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1775766546.488010:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098945024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766546.488012:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766546.488025:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4007 00000100:00080000:0.0:1775766546.488027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766546.488030:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766546.488053:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00080000:3.0:1775766546.488059:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766546.488064:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098945024:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775766546.488071:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766546.488093:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098945536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766546.488104:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098945152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766546.488145:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766546.488168:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766546/real 1775766546] req@ffff8b23864f3480 x1862022098945152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766546.488192:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098945152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766546.488207:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766546.488213:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766546.488220:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098945152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766546.488233:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098945280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766546.488318:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766546.488333:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766546/real 1775766546] req@ffff8b23864f2d80 x1862022098945280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766546.488348:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098945280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766546.488357:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766546.488360:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766546.488363:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098945280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766546.488368:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098945408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766546.488388:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766546.488396:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766546/real 1775766546] req@ffff8b23864f0700 x1862022098945408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766546.488405:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098945408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766546.488411:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766546.488413:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766546.488416:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098945408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766546.488429:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098945536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766546.488440:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766546.488446:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766546/real 1775766546] req@ffff8b23864f1180 x1862022098945536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766546.488495:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098945536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766546.488515:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766546.488534:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766546.488539:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098945536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766547.383674:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766548.407612:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766549.431632:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766550.455600:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766551.479732:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766551.607608:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766551.607620:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766551.607625:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766551.607635:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4013 00000100:00080000:3.0:1775766551.607648:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766551.607655:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766551.607715:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098945664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766551.607749:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766551.607753:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766551.607755:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766551.607759:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4013 00000100:00080000:3.0:1775766551.607763:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766551.607766:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766551.607775:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766551.607790:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098945792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766551.607803:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766551.607807:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766551.607810:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766551.607814:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4013 00000100:00080000:3.0:1775766551.607819:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766551.607821:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098945664:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766551.607826:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766551.607838:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766551.607850:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098945920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766551.607868:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766551.607872:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766551.607874:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766551.607878:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4013 00000100:00080000:3.0:1775766551.607882:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766551.607885:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766551.607891:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766551.607905:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766551.607906:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098946048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766551.607918:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766551.607922:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766551.607924:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766551.607929:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4013 00000100:00080000:3.0:1775766551.607933:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766551.607937:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766551.607942:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766551.607963:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766551/real 1775766551] req@ffff8b23864f3b80 x1862022098945664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766551.607963:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098946176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766551.608011:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098945664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766551.608027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766551.608031:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766551.608036:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098945664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766551.608047:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098945792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766551.608070:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766551.608082:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766551/real 1775766551] req@ffff8b23864f0000 x1862022098945792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766551.608093:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098945792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766551.608115:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766551.608120:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766551.608126:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098945792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766551.608162:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098945920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766551.608193:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766551.608231:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766551/real 1775766551] req@ffff8b23864f3100 x1862022098945920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766551.608247:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098945920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766551.608256:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766551.608259:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766551.608262:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098945920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766551.608269:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098946048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766551.608292:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766551.608302:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766551/real 1775766551] req@ffff8b23864f1c00 x1862022098946048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766551.608313:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098946048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766551.608320:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766551.608323:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766551.608327:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098946048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766551.608333:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098946176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766551.608348:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766551.608355:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766551/real 1775766551] req@ffff8b23864f1f80 x1862022098946176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766551.608366:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098946176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766551.608378:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766551.608383:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766551.608388:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098946176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766552.503646:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766553.527584:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766554.551696:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766555.575599:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766556.599623:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766556.727549:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766556.727557:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766556.727561:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766556.727570:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4018 00000100:00080000:0.0:1775766556.727580:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766556.727586:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766556.727622:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098946304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766556.727646:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766556.727650:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766556.727651:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766556.727654:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4018 00000100:00080000:0.0:1775766556.727656:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766556.727659:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766556.727668:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766556.727677:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098946432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766556.727684:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098946304:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766556.727688:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766556.727695:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766556.727697:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766556.727701:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4018 00000100:00080000:0.0:1775766556.727703:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766556.727706:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766556.727709:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766556.727718:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098946560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766556.727724:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766556.727727:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766556.727728:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766556.727731:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4018 00000100:00080000:0.0:1775766556.727733:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766556.727735:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766556.727738:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766556.727747:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098946688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766556.727753:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766556.727768:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766556.727772:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766556.727773:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766556.727777:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4018 00000100:00080000:0.0:1775766556.727780:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766556.727783:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766556.727786:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766556.727795:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766556/real 1775766556] req@ffff8b2382daed80 x1862022098946304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766556.727798:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098946816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766556.727829:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098946304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766556.727844:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766556.727847:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766556.727850:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098946304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766556.727874:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098946432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766556.727889:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766556.727897:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766556/real 1775766556] req@ffff8b2382daea00 x1862022098946432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766556.727915:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098946432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766556.727920:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766556.727923:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766556.727925:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098946432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766556.727928:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098946560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766556.727939:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766556.727944:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766556/real 1775766556] req@ffff8b2382dad500 x1862022098946560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766556.727956:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098946560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766556.727961:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766556.727963:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766556.727965:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098946560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766556.727968:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098946688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766556.727981:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766556.727991:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766556/real 1775766556] req@ffff8b2382dae680 x1862022098946688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766556.728003:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098946688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766556.728010:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766556.728014:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766556.728017:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098946688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766556.728039:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098946816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766556.728063:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766556.728076:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766556/real 1775766556] req@ffff8b2382dac380 x1862022098946816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766556.728106:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098946816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766556.728120:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766556.728123:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766556.728127:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098946816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766557.623567:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766558.647646:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766559.671605:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766560.695684:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766561.719738:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766561.847572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766561.847583:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766561.847587:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766561.847597:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4023 00000100:00080000:2.0:1775766561.847609:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766561.847616:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766561.847654:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022098946944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766561.847678:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766561.847681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766561.847682:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766561.847689:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4023 00000100:00080000:2.0:1775766561.847692:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766561.847695:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766561.847703:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766561.847713:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098947072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766561.847722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766561.847724:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766561.847727:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766561.847730:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098946944:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775766561.847731:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4023 00000100:00080000:2.0:1775766561.847735:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766561.847740:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766561.847743:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766561.847751:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098947200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766561.847758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766561.847761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766561.847762:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766561.847764:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4023 00000100:00080000:2.0:1775766561.847766:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766561.847768:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766561.847771:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766561.847778:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098947328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766561.847783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766561.847785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766561.847786:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766561.847789:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4023 00000100:00080000:2.0:1775766561.847792:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766561.847794:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766561.847797:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766561.847803:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775766561.847806:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098947456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766561.847845:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766561/real 1775766561] req@ffff8b23b9ea6300 x1862022098946944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766561.847860:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022098946944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766561.847868:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766561.847871:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766561.847875:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098946944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766561.847899:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098947072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766561.847925:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766561.847937:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766561/real 1775766561] req@ffff8b23b9ea7480 x1862022098947072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766561.847946:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022098947072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766561.847951:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766561.847954:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766561.847956:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098947072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766561.847961:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098947200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766561.847974:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766561.847980:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766561/real 1775766561] req@ffff8b23b9ea4e00 x1862022098947200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766561.847988:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022098947200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766561.847993:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766561.847995:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766561.847996:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098947200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766561.848000:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098947328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766561.848011:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766561.848016:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766561/real 1775766561] req@ffff8b23b9ea4380 x1862022098947328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766561.848023:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022098947328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766561.848028:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766561.848029:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766561.848031:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098947328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766561.848035:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098947456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766561.848045:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766561.848050:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766561/real 1775766561] req@ffff8b23b9ea7100 x1862022098947456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766561.848056:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022098947456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766561.848061:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766561.848063:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766561.848065:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098947456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766562.743694:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766563.767667:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766564.791703:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766565.815613:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766566.839641:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766566.967594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766566.967604:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766566.967609:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766566.967619:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4028 00000100:00080000:0.0:1775766566.967631:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766566.967638:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766566.967705:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098947584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766566.967731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766566.967735:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766566.967737:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766566.967740:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4028 00000100:00080000:0.0:1775766566.967743:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766566.967746:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766566.967755:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766566.967785:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098947712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766566.967798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766566.967802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766566.967805:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766566.967809:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4028 00000100:00080000:0.0:1775766566.967813:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766566.967834:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766566.967845:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098947584:192.168.202.128@tcp:250:kworker.0 00000008:00100000:0.0:1775766566.967851:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766566.967894:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098947840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766566.967910:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766566.967915:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766566.967918:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766566.967923:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4028 00000100:00080000:0.0:1775766566.967932:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766566.967936:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766566.967942:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766566.967947:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766566.967960:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098947968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766566.967989:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766566.967992:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766566.967994:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766566.967997:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4028 00000100:00080000:0.0:1775766566.968003:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766566.968007:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766566.968012:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766566.968023:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098948096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766566.968100:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766566/real 1775766566] req@ffff8b2382dac000 x1862022098947584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766566.968120:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098947584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766566.968129:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766566.968133:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766566.968137:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098947584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766566.968144:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098947712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766566.968167:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766566.968178:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766566/real 1775766566] req@ffff8b2382daf100 x1862022098947712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766566.968188:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098947712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766566.968195:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766566.968197:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766566.968200:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098947712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766566.968223:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098947840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766566.968237:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766566.968247:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766566/real 1775766566] req@ffff8b2382daca80 x1862022098947840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766566.968263:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098947840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766566.968273:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766566.968277:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766566.968281:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098947840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766566.968290:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098947968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766566.968317:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766566.968331:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766566/real 1775766566] req@ffff8b2382dad880 x1862022098947968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766566.968344:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098947968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766566.968351:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766566.968354:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766566.968357:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098947968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766566.968362:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098948096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766566.968380:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766566.968387:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766566/real 1775766566] req@ffff8b2382dac380 x1862022098948096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766566.968396:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098948096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766566.968403:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766566.968406:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766566.968410:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098948096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766567.863583:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766568.887661:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766569.911612:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766570.935770:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766571.959721:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766572.087572:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766572.087602:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766572.087609:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766572.087622:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4033 00000100:00080000:3.0:1775766572.087638:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766572.087647:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766572.087701:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098948224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766572.087772:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766572.087777:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766572.087781:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766572.087788:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4033 00000100:00080000:3.0:1775766572.087796:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766572.087801:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766572.087817:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766572.087831:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098948224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766572.087851:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098948352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766572.087893:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766572.087898:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766572.087900:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766572.087917:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4033 00000100:00080000:3.0:1775766572.087927:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766572.087934:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766572.087942:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766572.087956:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098948480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766572.088006:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766572.088012:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766572.088017:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766572.088020:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766572.088057:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4033 00000100:00080000:3.0:1775766572.088062:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766572.088066:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766572.088072:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766572.088086:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098948608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766572.088091:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766572/real 1775766572] req@ffff8b23864f0a80 x1862022098948224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766572.088123:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766572.088131:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766572.088132:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766572.088136:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4033 00000100:00080000:3.0:1775766572.088141:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766572.088142:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098948224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766572.088147:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766572.088156:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766572.088169:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098948736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766572.088171:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766572.088192:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766572.088199:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098948224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766572.088228:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098948352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766572.088249:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766572.088259:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766572/real 1775766572] req@ffff8b23864f2680 x1862022098948352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766572.088268:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098948352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766572.088273:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766572.088276:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766572.088279:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098948352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766572.088283:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098948480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766572.088294:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766572.088319:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766572/real 1775766572] req@ffff8b23864f0380 x1862022098948480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766572.088328:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098948480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766572.088335:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766572.088338:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766572.088340:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098948480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766572.088345:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098948608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766572.088380:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766572.088387:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766572/real 1775766572] req@ffff8b23864f1f80 x1862022098948608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766572.088394:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098948608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766572.088399:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766572.088400:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766572.088402:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098948608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766572.088406:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098948736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766572.088417:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766572.088423:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766572/real 1775766572] req@ffff8b23864f1c00 x1862022098948736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766572.088430:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098948736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766572.088435:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766572.088436:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766572.088438:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098948736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766572.983722:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766574.007886:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766575.031700:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766576.055655:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766577.079658:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766577.207702:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766577.207784:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766577.207793:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766577.207812:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4038 00000100:00080000:0.0:1775766577.207834:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766577.207866:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766577.207969:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098948864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766577.208034:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766577.208039:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766577.208042:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766577.208056:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4038 00000100:00080000:0.0:1775766577.208064:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766577.208070:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766577.208085:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766577.208107:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098948992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766577.208110:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098948864:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766577.208148:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766577.208152:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766577.208154:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766577.208178:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4038 00000100:00080000:0.0:1775766577.208182:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766577.208187:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766577.208192:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766577.208234:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098949120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766577.208247:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766577.208252:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766577.208254:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766577.208258:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4038 00000100:00080000:0.0:1775766577.208264:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775766577.208272:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766577.208278:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766577.208285:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766577.208300:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098949248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766577.208309:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766577.208313:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766577.208315:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766577.208320:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4038 00000100:00080000:0.0:1775766577.208345:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766577.208349:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766577.208354:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766577.208367:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098949376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766577.208385:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766577/real 1775766577] req@ffff8b2382dae680 x1862022098948864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766577.208428:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098948864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766577.208489:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766577.208496:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766577.208503:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098948864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766577.210341:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098948992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766577.210373:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766577.210387:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766577/real 1775766577] req@ffff8b2382dad500 x1862022098948992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766577.210400:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098948992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766577.210408:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766577.210411:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766577.210414:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098948992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766577.210421:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098949120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766577.210475:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766577.210486:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766577/real 1775766577] req@ffff8b2382daea00 x1862022098949120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766577.210497:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098949120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766577.210505:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766577.210508:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766577.210512:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098949120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766577.210519:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098949248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766577.210552:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766577.210562:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766577/real 1775766577] req@ffff8b2382daed80 x1862022098949248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766577.210573:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098949248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766577.210598:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766577.210603:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766577.210607:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098949248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766577.210615:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098949376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766577.210640:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766577.210650:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766577/real 1775766577] req@ffff8b2382dadf80 x1862022098949376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766577.210664:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098949376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766577.210673:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766577.210677:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766577.210681:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098949376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766578.103691:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766579.127605:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766580.151659:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766581.175647:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766582.199669:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766582.327587:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766582.327597:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766582.327601:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766582.327612:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4043 00000100:00080000:0.0:1775766582.327623:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766582.327630:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766582.327687:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098949504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766582.327713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766582.327716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766582.327718:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766582.327722:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4043 00000100:00080000:0.0:1775766582.327724:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766582.327728:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766582.327737:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766582.327748:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098949632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766582.327758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766582.327782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766582.327785:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766582.327791:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4043 00000100:00080000:0.0:1775766582.327796:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766582.327801:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766582.327807:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766582.327822:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098949760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766582.327837:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098949504:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766582.327839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766582.327852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766582.327857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766582.327923:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4043 00000100:00080000:0.0:1775766582.327927:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766582.327931:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766582.327936:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766582.327953:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098949888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766582.327962:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766582.327966:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766582.327968:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766582.327971:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4043 00000100:00080000:0.0:1775766582.327975:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766582.327978:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766582.327982:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766582.327992:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098950016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766582.328017:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098949632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766582.328052:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098949760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766582.328074:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098949888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766582.328094:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098950016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766583.223682:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766584.247724:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766585.271767:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775766585.399938:0:2365:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775766585.399997:0:2365:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775766585.400016:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766585.400059:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766585.400066:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766585.400073:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766585.400084:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:3.0:1775766585.400126:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766582/real 1775766585] req@ffff8b2382dafb80 x1862022098949760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766585.400166:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098949760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766585.400183:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766585.400191:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766585.400197:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098949760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766585.400207:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766582/real 1775766585] req@ffff8b2382dac700 x1862022098949888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766585.400223:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098949888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766585.400234:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766585.400238:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766585.400242:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098949888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766585.400256:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766582/real 1775766585] req@ffff8b2382dadf80 x1862022098950016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766585.400271:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098950016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766585.400282:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766585.400287:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766585.400292:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098950016:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766585.400301:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766582/real 1775766585] req@ffff8b2382daf480 x1862022098949504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766585.400318:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098949504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766585.400329:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766585.400334:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766585.400338:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098949504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766585.400348:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766582/real 1775766585] req@ffff8b2382dace00 x1862022098949632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766585.400360:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098949632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766585.400366:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766585.400372:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766585.400376:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098949632:192.168.202.128@tcp:38:kworker.0 00010000:00010000:1.0:1775766586.295656:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766587.319638:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766587.447565:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766587.447573:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766587.447577:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766587.447586:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4048 00000100:00080000:2.0:1775766587.447595:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766587.447601:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766587.447634:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098950144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766587.447657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766587.447659:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766587.447661:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766587.447663:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4048 00000100:00080000:2.0:1775766587.447666:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766587.447668:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766587.447676:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766587.447683:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022098950272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766587.447691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766587.447696:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766587.447697:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766587.447701:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4048 00000100:00080000:2.0:1775766587.447703:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766587.447706:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766587.447709:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766587.447715:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098950400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766587.447722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766587.447727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1775766587.447727:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098950144:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775766587.447729:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766587.447737:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4048 00000100:00080000:2.0:1775766587.447739:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766587.447741:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766587.447744:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766587.447754:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098950528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766587.447763:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766587.447767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766587.447769:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766587.447772:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4048 00000100:00080000:2.0:1775766587.447785:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766587.447789:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775766587.447796:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:2.0:1775766587.447797:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766587.447812:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098950656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766587.447873:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766587/real 1775766587] req@ffff8b23b9ea4a80 x1862022098950144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766587.447895:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022098950144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766587.447908:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766587.447913:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766587.447917:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098950144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766587.447925:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098950272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766587.447946:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766587.447956:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766587/real 1775766587] req@ffff8b23b9ea5880 x1862022098950272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766587.447964:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022098950272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766587.447970:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766587.447972:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766587.447975:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098950272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766587.447993:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098950400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766587.448004:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766587.448011:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766587/real 1775766587] req@ffff8b23b9ea4000 x1862022098950400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766587.448018:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022098950400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766587.448023:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766587.448025:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766587.448027:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098950400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766587.448031:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098950528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766587.448041:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766587.448046:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766587/real 1775766587] req@ffff8b23b9ea6a00 x1862022098950528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766587.448053:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022098950528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766587.448058:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766587.448059:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766587.448061:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098950528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766587.448064:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098950656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766587.448073:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766587.448078:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766587/real 1775766587] req@ffff8b23b9ea4700 x1862022098950656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766587.448084:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022098950656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766587.448090:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766587.448093:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766587.448095:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098950656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766588.343673:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766589.367675:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766590.391679:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766591.415776:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766592.439688:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766592.567633:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766592.567645:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766592.567651:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766592.567661:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4053 00000100:00080000:0.0:1775766592.567672:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766592.567698:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766592.567737:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098950784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766592.567765:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766592.567782:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766592.567784:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766592.567788:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4053 00000100:00080000:0.0:1775766592.567793:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766592.567798:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766592.567808:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766592.567833:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098950912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766592.567842:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766592.567847:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766592.567849:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766592.567849:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098950784:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766592.567855:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4053 00000100:00080000:0.0:1775766592.567861:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766592.567863:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766592.567867:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766592.567876:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098951040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766592.567882:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766592.567885:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766592.567887:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766592.567889:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4053 00000100:00080000:0.0:1775766592.567892:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766592.567895:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766592.567897:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766592.567905:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098951168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766592.567911:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766592.567913:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766592.567914:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766592.567916:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4053 00000100:00080000:0.0:1775766592.567919:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775766592.567924:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766592.567925:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766592.567930:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766592.567940:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098951296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766592.567975:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766592/real 1775766592] req@ffff8b2382daed80 x1862022098950784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766592.567997:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098950784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766592.568010:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766592.568013:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766592.568016:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098950784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766592.568043:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098950912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766592.568066:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766592.568079:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766592/real 1775766592] req@ffff8b2382daea00 x1862022098950912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766592.568094:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098950912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766592.568103:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766592.568108:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766592.568111:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098950912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766592.568118:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098951040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766592.568140:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766592.568152:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766592/real 1775766592] req@ffff8b2382dad500 x1862022098951040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766592.568163:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098951040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766592.568172:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766592.568176:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766592.568180:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098951040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766592.568186:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098951168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766592.568231:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766592.568246:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766592/real 1775766592] req@ffff8b2382dae680 x1862022098951168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766592.568263:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098951168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766592.568289:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766592.568292:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766592.568296:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098951168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766592.568302:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098951296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766592.568321:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766592.568331:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766592/real 1775766592] req@ffff8b2382dac380 x1862022098951296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766592.568363:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098951296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766592.568375:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766592.568380:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766592.568384:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098951296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766593.463605:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766594.487648:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766595.511587:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766596.535693:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766597.559654:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766597.687578:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766597.687588:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766597.687594:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766597.687606:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4059 00000100:00080000:0.0:1775766597.687623:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766597.687633:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766597.687741:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098951424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766597.687789:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766597.687795:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766597.687797:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766597.687803:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4059 00000100:00080000:0.0:1775766597.687809:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766597.687815:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766597.687828:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766597.687854:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098951552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766597.687870:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098951424:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766597.687892:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766597.687901:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766597.687905:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766597.687916:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4059 00000100:00080000:0.0:1775766597.687925:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766597.687934:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766597.687946:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766597.687969:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766597.687977:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098951680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766597.687994:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766597.687999:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766597.688002:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766597.688008:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4059 00000100:00080000:0.0:1775766597.688016:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766597.688022:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766597.688030:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766597/real 1775766597] req@ffff8b2382dad880 x1862022098951424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775766597.688033:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766597.688067:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098951808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766597.688078:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098951424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766597.688088:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766597.688103:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766597.688109:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766597.688117:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766597.688119:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4059 00000100:00080000:0.0:1775766597.688131:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766597.688137:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00080000:0.0:1775766597.688144:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766597.688152:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098951424:192.168.202.128@tcp:250:kworker.0 00000008:00100000:0.0:1775766597.688153:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766597.688194:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098951936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766597.688199:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098951552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766597.688242:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766597.688262:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766597/real 1775766597] req@ffff8b2382daca80 x1862022098951552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766597.688279:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098951552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766597.688290:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766597.688294:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766597.688298:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098951552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766597.688306:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098951680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766597.688324:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766597.688335:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766597/real 1775766597] req@ffff8b2382daf100 x1862022098951680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766597.688349:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098951680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766597.688358:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766597.688378:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766597.688382:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098951680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766597.688390:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098951808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766597.688422:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766597.688442:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766597/real 1775766597] req@ffff8b2382dac000 x1862022098951808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766597.688523:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098951808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766597.688536:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766597.688540:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766597.688544:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098951808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766597.688569:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098951936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766597.688599:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766597.688612:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766597/real 1775766597] req@ffff8b2382dac380 x1862022098951936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766597.688626:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098951936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766597.688636:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766597.688640:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766597.688643:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098951936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766598.583638:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766599.607634:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766600.631597:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766601.655651:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766602.679646:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766602.807574:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766602.807587:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766602.807593:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766602.807606:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4064 00000100:00080000:0.0:1775766602.807619:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766602.807628:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766602.807735:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098952064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766602.807774:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766602.807784:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766602.807787:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766602.807792:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4064 00000100:00080000:0.0:1775766602.807798:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766602.807803:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766602.807816:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766602.807832:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098952064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766602.807838:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098952192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766602.807855:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766602.807859:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766602.807862:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766602.807877:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4064 00000100:00080000:0.0:1775766602.807883:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766602.807888:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766602.807895:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766602.807901:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766602.807913:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098952320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766602.807935:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766602.807940:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766602.807943:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766602.807946:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766602/real 1775766602] req@ffff8b2382dae680 x1862022098952064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766602.807952:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4064 00000100:00080000:0.0:1775766602.807963:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766602.807970:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766602.807974:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098952064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775766602.807979:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775766602.807989:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766602.807995:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766602.808003:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098952448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766602.808003:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098952064:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766602.808022:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766602.808027:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766602.808029:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766602.808034:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098952192:192.168.202.128@tcp:38:kworker.0 00000100:00080000:0.0:1775766602.808037:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4064 00000100:00080000:0.0:1775766602.808044:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766602.808049:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766602.808055:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766602.808062:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766602.808073:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098952576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766602.808078:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766602/real 1775766602] req@ffff8b2382dad500 x1862022098952192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766602.808120:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098952192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766602.808128:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766602.808132:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766602.808136:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098952192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766602.808142:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098952320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766602.808158:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766602.808165:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766602/real 1775766602] req@ffff8b2382daea00 x1862022098952320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766602.808175:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098952320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766602.808181:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766602.808184:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766602.808186:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098952320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766602.808191:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098952448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766602.808204:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766602.808211:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766602/real 1775766602] req@ffff8b2382daed80 x1862022098952448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766602.808220:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098952448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766602.808225:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766602.808228:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766602.808230:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098952448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766602.808242:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098952576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766602.808254:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766602.808261:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766602/real 1775766602] req@ffff8b2382dace00 x1862022098952576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766602.808269:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098952576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766602.808276:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766602.808278:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766602.808280:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098952576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766603.703710:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766604.727681:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766605.751682:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766606.775578:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766607.799632:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766607.927649:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766607.927668:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766607.927672:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766607.927682:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4069 00000100:00080000:3.0:1775766607.927691:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766607.927696:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766607.927735:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098952704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766607.927772:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766607.927777:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766607.927779:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766607.927783:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4069 00000100:00080000:3.0:1775766607.927787:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766607.927791:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766607.927800:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766607.927810:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098952832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766607.927822:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766607.927825:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766607.927827:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766607.927832:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4069 00000100:00080000:3.0:1775766607.927835:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766607.927841:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766607.927847:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766607.927860:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098952960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766607.927875:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098952704:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766607.927908:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766607.927913:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766607.927915:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766607.927921:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4069 00000100:00080000:3.0:1775766607.927926:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766607.927938:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766607.927947:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766607.927959:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098953088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766607.927966:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766607.927969:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766607.927971:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766607.927974:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4069 00000100:00080000:3.0:1775766607.927976:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766607.927979:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766607.927983:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766607.927991:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766607.927996:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098953216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766607.928059:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766607/real 1775766607] req@ffff8b23864f3100 x1862022098952704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766607.928093:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098952704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766607.928107:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766607.928113:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766607.928119:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098952704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766607.928129:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098952832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766607.928155:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766607.928169:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766607/real 1775766607] req@ffff8b23864f0000 x1862022098952832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766607.928208:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098952832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766607.928224:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766607.928230:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766607.928237:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098952832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766607.928283:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098952960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766607.928321:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766607.928336:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766607/real 1775766607] req@ffff8b23864f3b80 x1862022098952960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766607.928351:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098952960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766607.928369:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766607.928373:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766607.928377:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098952960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766607.928384:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098953088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766607.928405:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766607.928416:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766607/real 1775766607] req@ffff8b23864f1180 x1862022098953088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766607.928428:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098953088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766607.928437:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766607.928441:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766607.928444:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098953088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766607.928450:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098953216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766607.928600:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766607.928647:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766607/real 1775766607] req@ffff8b23864f0700 x1862022098953216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766607.928681:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098953216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766607.928691:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766607.928695:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766607.928700:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098953216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766608.823609:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766609.847650:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766610.871808:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766611.895693:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766612.919794:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766613.047648:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766613.047661:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766613.047667:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766613.047679:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4074 00000100:00080000:2.0:1775766613.047694:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766613.047702:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766613.047757:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098953344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766613.047802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766613.047808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766613.047810:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766613.047817:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4074 00000100:00080000:2.0:1775766613.047823:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766613.047830:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766613.047845:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766613.047859:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098953344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775766613.047869:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098953472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766613.047890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766613.047895:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766613.047898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766613.047906:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4074 00000100:00080000:2.0:1775766613.047914:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766613.047921:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766613.047929:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766613.047934:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775766613.047945:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022098953600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766613.047957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766613.047961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766613.047964:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766613.047970:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4074 00000100:00080000:2.0:1775766613.047975:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766613.047981:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766613/real 1775766613] req@ffff8b23b9ea5500 x1862022098953344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766613.047982:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766613.047994:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766613.048015:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098953728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766613.048026:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022098953344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766613.048031:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766613.048041:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766613.048045:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766613.048060:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4074 00000100:00080000:2.0:1775766613.048070:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766613.048076:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:0.0:1775766613.048077:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1775766613.048084:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775766613.048088:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766613.048094:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098953344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775766613.048099:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098953856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766613.048124:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098953472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766613.048147:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766613.048158:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766613/real 1775766613] req@ffff8b23b9ea5c00 x1862022098953472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766613.048167:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022098953472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766613.048173:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766613.048175:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766613.048177:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098953472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766613.048181:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098953600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766613.048192:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766613.048199:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766613/real 1775766613] req@ffff8b23b9ea6d80 x1862022098953600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766613.048206:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022098953600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766613.048211:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766613.048213:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766613.048214:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098953600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766613.048218:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098953728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766613.048230:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766613.048236:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766613/real 1775766613] req@ffff8b23b9ea6680 x1862022098953728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766613.048243:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022098953728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766613.048248:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766613.048250:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766613.048252:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098953728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766613.048255:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098953856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766613.048291:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766613.048299:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766613/real 1775766613] req@ffff8b23b9ea5f80 x1862022098953856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766613.048307:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022098953856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766613.048312:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766613.048315:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766613.048317:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098953856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766613.943675:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766614.967591:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766615.991609:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766617.015636:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766618.039590:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766618.167546:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766618.167554:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766618.167558:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766618.167566:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4079 00000100:00080000:3.0:1775766618.167575:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766618.167580:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766618.167612:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098953984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766618.167634:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766618.167637:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766618.167638:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766618.167641:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4079 00000100:00080000:3.0:1775766618.167643:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766618.167646:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766618.167654:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766618.167663:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098954112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766618.167672:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098953984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766618.167687:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766618.167690:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766618.167691:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766618.167695:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4079 00000100:00080000:3.0:1775766618.167697:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766618.167700:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766618.167704:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766618.167714:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098954240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766618.167721:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766618.167723:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766618.167724:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766618.167727:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4079 00000800:00000100:0.0:1775766618.167731:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766618.167731:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766618.167736:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766618.167740:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766618.167749:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098954368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766618.167755:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766618.167758:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766618.167759:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766618.167762:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4079 00000100:00080000:3.0:1775766618.167765:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766618.167768:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766618/real 1775766618] req@ffff8b23864f2d80 x1862022098953984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766618.167771:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766618.167777:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766618.167788:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098954496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766618.167790:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098953984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766618.167815:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766618.167819:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766618.167822:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098953984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766618.167843:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098954112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766618.167858:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766618.167866:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766618/real 1775766618] req@ffff8b23864f3480 x1862022098954112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766618.167874:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098954112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766618.167879:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766618.167881:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766618.167883:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098954112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766618.167887:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098954240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766618.167898:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766618.167904:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766618/real 1775766618] req@ffff8b23864f2a00 x1862022098954240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766618.167911:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098954240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766618.167915:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766618.167917:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766618.167919:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098954240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766618.167922:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098954368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766618.167933:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766618.167938:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766618/real 1775766618] req@ffff8b23864f0700 x1862022098954368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766618.167945:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098954368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766618.167949:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766618.167951:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766618.167952:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098954368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766618.167956:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098954496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766618.167965:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766618.167969:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766618/real 1775766618] req@ffff8b23864f1180 x1862022098954496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766618.167977:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098954496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766618.167982:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766618.167983:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766618.167985:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098954496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766619.063645:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766620.087650:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766621.111650:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766622.135654:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766623.159619:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766623.287545:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766623.287553:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766623.287557:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766623.287565:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4084 00000100:00080000:3.0:1775766623.287575:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766623.287580:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766623.287616:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098954624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766623.287645:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766623.287649:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766623.287651:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766623.287655:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4084 00000100:00080000:3.0:1775766623.287659:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766623.287664:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766623.287675:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766623.287688:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098954624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766623.287695:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098954752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766623.287710:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766623.287714:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766623.287716:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766623.287721:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4084 00000100:00080000:3.0:1775766623.287725:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766623.287728:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766623.287733:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766623.287744:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098954880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766623.287751:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766623.287761:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766623.287764:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766623.287766:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766623.287770:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4084 00000100:00080000:3.0:1775766623.287772:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766623.287774:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766623.287778:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766623.287786:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098955008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766623.287790:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766623/real 1775766623] req@ffff8b23864f3b80 x1862022098954624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766623.287799:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766623.287806:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766623.287809:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766623.287814:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4084 00000100:00080000:3.0:1775766623.287817:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766623.287822:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098954624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766623.287822:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766623.287826:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775766623.287836:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1775766623.287840:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098955136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766623.287843:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766623.287849:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098954624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766623.287876:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098954752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766623.287901:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766623.287914:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766623/real 1775766623] req@ffff8b23864f0000 x1862022098954752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766623.287928:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098954752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766623.287942:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766623.287945:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766623.287947:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098954752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766623.287952:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098954880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766623.287969:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766623.287976:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766623/real 1775766623] req@ffff8b23864f3100 x1862022098954880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766623.287984:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098954880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766623.287990:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766623.287992:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766623.287994:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098954880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766623.287998:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098955008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766623.288014:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766623.288024:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766623/real 1775766623] req@ffff8b23864f1c00 x1862022098955008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766623.288062:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098955008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766623.288072:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766623.288075:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766623.288078:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098955008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766623.288084:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098955136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766623.288104:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766623.288116:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766623/real 1775766623] req@ffff8b23864f1f80 x1862022098955136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766623.288129:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098955136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766623.288138:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766623.288142:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766623.288145:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098955136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766624.183604:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766625.207701:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766626.231634:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766627.255603:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766628.279654:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766628.407572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766628.407580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766628.407584:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766628.407593:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4089 00000100:00080000:2.0:1775766628.407602:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766628.407607:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766628.407660:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022098955264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766628.407687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766628.407690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766628.407692:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766628.407694:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4089 00000100:00080000:2.0:1775766628.407697:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766628.407699:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766628.407707:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766628.407717:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098955392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766628.407727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766628.407730:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766628.407731:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766628.407735:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4089 00000100:00080000:2.0:1775766628.407737:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766628.407740:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766628.407744:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766628.407755:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098955520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766628.407759:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098955264:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775766628.407779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766628.407782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766628.407783:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766628.407786:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4089 00000100:00080000:2.0:1775766628.407788:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766628.407791:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766628.407795:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766628.407805:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022098955648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766628.407811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766628.407814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766628.407815:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766628.407818:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4089 00000100:00080000:2.0:1775766628.407820:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766628.407823:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766628.407827:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766628.407833:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098955776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766628.407869:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766628.407925:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766628/real 1775766628] req@ffff8b23b9ea7b80 x1862022098955264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766628.407941:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022098955264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766628.407950:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766628.407954:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766628.407957:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098955264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766628.407963:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098955392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766628.407979:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766628.407988:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766628/real 1775766628] req@ffff8b23b9ea5f80 x1862022098955392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766628.407995:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022098955392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766628.408001:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766628.408003:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766628.408005:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098955392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766628.408029:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098955520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766628.408040:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766628.408046:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766628/real 1775766628] req@ffff8b23b9ea6680 x1862022098955520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766628.408054:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022098955520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766628.408059:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766628.408061:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766628.408063:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098955520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766628.408066:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098955648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766628.408077:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766628.408082:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766628/real 1775766628] req@ffff8b23b9ea6d80 x1862022098955648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766628.408090:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022098955648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766628.408095:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766628.408097:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766628.408100:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098955648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766628.408103:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098955776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766628.408113:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766628.408119:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766628/real 1775766628] req@ffff8b23b9ea5c00 x1862022098955776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766628.408126:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022098955776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766628.408131:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766628.408133:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766628.408134:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098955776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766629.303674:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766630.327649:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766631.351605:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766632.375709:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766633.399627:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766633.527619:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766633.527633:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766633.527639:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766633.527653:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4094 00000100:00080000:0.0:1775766633.527668:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766633.527677:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766633.527731:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098955904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766633.527775:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766633.527782:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766633.527812:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766633.527820:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4094 00000100:00080000:0.0:1775766633.527828:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766633.527835:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766633.527841:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098955904:192.168.202.128@tcp:250:kworker.0 00000008:00100000:0.0:1775766633.527851:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766633.527878:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098956032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766633.527897:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766633.527914:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766633.527918:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775766633.527928:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766633.527929:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4094 00000100:00080000:0.0:1775766633.527940:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766633.527948:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766633.527958:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766633.527978:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766633/real 1775766633] req@ffff8b2382daf480 x1862022098955904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766633.527984:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098956160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766633.528013:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766633.528016:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098955904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766633.528026:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766633.528032:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766633.528038:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766633.528044:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4094 00000100:00080000:3.0:1775766633.528048:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00080000:0.0:1775766633.528056:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766633.528056:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098955904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766633.528070:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766633.528080:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766633.528093:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098956032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766633.528103:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098956288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766633.528120:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766633.528123:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766633.528129:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766633.528132:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766633.528141:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4094 00000100:00100000:3.0:1775766633.528142:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766633/real 1775766633] req@ffff8b2382dadf80 x1862022098956032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766633.528150:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766633.528163:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766633.528175:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766633.528179:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098956032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766633.528196:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1775766633.528197:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098956416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766633.528208:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766633.528221:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098956032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766633.528234:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098956160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766633.528272:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766633.528287:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766633/real 1775766633] req@ffff8b2382dac700 x1862022098956160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766633.528309:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098956160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766633.528318:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766633.528322:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766633.528338:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098956160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766633.528377:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098956288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766633.528415:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766633.528427:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766633/real 1775766633] req@ffff8b2382dafb80 x1862022098956288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766633.528440:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098956288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766633.528448:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766633.528496:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766633.528501:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098956288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766633.528510:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098956416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766633.528541:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766633.528557:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766633/real 1775766633] req@ffff8b2382daf480 x1862022098956416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766633.528575:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098956416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766633.528597:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766633.528619:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766633.528624:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098956416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766634.423627:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766635.447604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766636.471610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766637.495613:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766638.519602:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766638.647555:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766638.647562:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766638.647566:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766638.647575:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4100 00000100:00080000:3.0:1775766638.647584:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766638.647590:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766638.647627:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098956544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766638.647661:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766638.647664:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766638.647665:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766638.647668:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4100 00000100:00080000:3.0:1775766638.647670:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766638.647672:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766638.647680:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766638.647688:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098956672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766638.647700:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766638.647703:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766638.647704:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766638.647706:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4100 00000100:00080000:3.0:1775766638.647708:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766638.647711:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766638.647714:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766638.647722:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098956800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766638.647728:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766638.647731:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766638.647732:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766638.647735:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4100 00000100:00080000:3.0:1775766638.647736:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766638.647739:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766638.647743:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766638.647745:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098956544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766638.647753:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098956928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766638.647759:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766638.647761:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766638.647762:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766638.647764:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4100 00000100:00080000:3.0:1775766638.647767:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766638.647770:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766638.647773:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766638.647791:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098957056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766638.647828:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766638.647864:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766638/real 1775766638] req@ffff8b23864f0380 x1862022098956544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766638.647879:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098956544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766638.647887:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766638.647889:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766638.647892:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098956544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766638.647897:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098956672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766638.647914:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766638.647921:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766638/real 1775766638] req@ffff8b23864f2680 x1862022098956672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766638.647929:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098956672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766638.647934:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766638.647936:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766638.647939:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098956672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766638.647942:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098956800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766638.647955:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766638.647961:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766638/real 1775766638] req@ffff8b23864f0a80 x1862022098956800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766638.647968:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098956800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766638.647973:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766638.647974:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766638.647976:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098956800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766638.647994:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098956928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766638.648005:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766638.648011:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766638/real 1775766638] req@ffff8b23864f1f80 x1862022098956928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766638.648022:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098956928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766638.648027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766638.648029:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766638.648030:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098956928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766638.648034:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098957056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766638.648044:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766638.648060:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766638/real 1775766638] req@ffff8b23864f1c00 x1862022098957056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766638.648067:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098957056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766638.648072:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766638.648074:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766638.648076:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098957056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766639.543625:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766640.567664:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766641.591644:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766642.615605:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766643.639756:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766643.767619:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766643.767628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766643.767633:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766643.767641:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4105 00000100:00080000:2.0:1775766643.767650:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766643.767656:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766643.767698:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098957184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766643.767722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766643.767725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766643.767726:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766643.767729:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4105 00000100:00080000:2.0:1775766643.767731:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766643.767733:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766643.767741:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766643.767749:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098957312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766643.767757:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766643.767760:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766643.767762:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766643.767766:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4105 00000100:00080000:2.0:1775766643.767768:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766643.767770:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766643.767774:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766643.767779:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098957184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775766643.767784:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098957440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766643.767796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766643.767798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766643.767799:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766643.767802:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4105 00000100:00080000:2.0:1775766643.767803:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766643.767805:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766643.767808:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766643.767815:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098957568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766643.767821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766643.767824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766643.767825:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766643.767827:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4105 00000100:00080000:2.0:1775766643.767829:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766643.767831:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766643.767835:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766643.767841:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022098957696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766643.767848:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766643.767894:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766643/real 1775766643] req@ffff8b23b9ea5500 x1862022098957184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766643.767926:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022098957184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766643.767939:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766643.767942:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766643.767945:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098957184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766643.767951:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098957312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766643.767970:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766643.767980:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766643/real 1775766643] req@ffff8b23b9ea4700 x1862022098957312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766643.767988:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022098957312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766643.767993:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766643.767996:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766643.767998:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098957312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766643.768015:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098957440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766643.768026:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766643.768032:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766643/real 1775766643] req@ffff8b23b9ea6a00 x1862022098957440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766643.768039:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022098957440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766643.768044:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766643.768046:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766643.768048:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098957440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766643.768052:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098957568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766643.768062:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766643.768067:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766643/real 1775766643] req@ffff8b23b9ea4000 x1862022098957568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766643.768075:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022098957568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766643.768079:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766643.768081:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766643.768083:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098957568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766643.768086:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098957696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766643.768097:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766643.768102:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766643/real 1775766643] req@ffff8b23b9ea5880 x1862022098957696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766643.768109:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022098957696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766643.768114:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766643.768116:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766643.768118:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098957696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766644.663631:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766645.687619:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766646.711620:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766647.735687:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766648.759715:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766648.887641:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766648.887658:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766648.887664:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766648.887679:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4110 00000100:00080000:3.0:1775766648.887696:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766648.887705:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766648.887755:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098957824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766648.887789:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766648.887792:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766648.887794:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766648.887797:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4110 00000100:00080000:3.0:1775766648.887801:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766648.887804:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766648.887815:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766648.887848:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098957952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766648.887861:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766648.887864:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766648.887865:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766648.887869:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4110 00000100:00080000:3.0:1775766648.887872:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766648.887877:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766648.887881:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766648.887897:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098957824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766648.887905:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098958080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766648.887915:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766648.887919:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766648.887920:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766648.887923:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4110 00000100:00080000:3.0:1775766648.887926:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766648.887928:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766648.887932:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766648.887942:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098958208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766648.887950:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766648.887953:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766648.887954:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766648.887959:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4110 00000100:00080000:3.0:1775766648.887961:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766648.887964:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766648.887968:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766648.887977:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098958336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766648.888088:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098957952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766648.888166:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098958080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766648.888257:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098958208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766648.888310:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098958336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766649.783749:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766650.807674:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766651.831850:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775766651.959980:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775766651.960044:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766651.960068:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766651.960119:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766651.960133:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766651.960183:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775766651.960196:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766648/real 1775766651] req@ffff8b23864f3b80 x1862022098958080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1775766651.960216:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775766651.960260:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098958080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766651.960275:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766651.960279:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766651.960284:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098958080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766651.960291:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766648/real 1775766651] req@ffff8b23864f1180 x1862022098958208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766651.960300:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098958208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766651.960306:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766651.960309:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766651.960312:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098958208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766651.960317:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766648/real 1775766651] req@ffff8b23864f0700 x1862022098958336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766651.960324:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098958336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766651.960335:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766651.960338:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766651.960340:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098958336:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766651.960345:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766648/real 1775766651] req@ffff8b23864f3100 x1862022098957824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766651.960353:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098957824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766651.960361:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766651.960364:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766651.960367:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098957824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766651.960375:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766648/real 1775766651] req@ffff8b23864f0000 x1862022098957952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766651.960387:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098957952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766651.960396:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766651.960400:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766651.960404:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098957952:192.168.202.128@tcp:38:kworker.0 00010000:00010000:3.0:1775766652.855612:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766653.879591:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766654.007543:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766654.007551:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766654.007555:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766654.007563:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4115 00000100:00080000:1.0:1775766654.007573:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766654.007579:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766654.007613:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098958464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766654.007635:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766654.007638:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766654.007639:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766654.007642:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4115 00000100:00080000:1.0:1775766654.007644:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766654.007646:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766654.007654:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766654.007663:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098958592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766654.007673:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098958464:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766654.007680:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766654.007683:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766654.007684:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766654.007688:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4115 00000100:00080000:1.0:1775766654.007690:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766654.007692:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766654.007696:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766654.007709:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098958720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766654.007717:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766654.007721:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766654.007723:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766654.007727:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4115 00000100:00080000:1.0:1775766654.007731:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775766654.007737:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775766654.007739:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766654.007747:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766654.007761:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098958848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766654.007770:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766654.007774:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766654.007776:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766654.007781:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4115 00000100:00080000:1.0:1775766654.007786:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766654.007788:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766654/real 1775766654] req@ffff8b23864fc380 x1862022098958464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766654.007793:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766654.007803:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766654.007819:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098958976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766654.007820:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098958464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766654.007851:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766654.007858:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766654.007863:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098958464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766654.007890:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098958592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766654.007917:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766654.007931:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766654/real 1775766654] req@ffff8b23864ff800 x1862022098958592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766654.007944:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098958592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766654.007951:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766654.007955:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766654.007958:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098958592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766654.007964:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098958720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766654.007984:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766654.007995:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766654/real 1775766654] req@ffff8b23864fdf80 x1862022098958720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766654.008007:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098958720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766654.008022:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766654.008026:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766654.008030:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098958720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766654.008037:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098958848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766654.008066:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766654.008079:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766654/real 1775766654] req@ffff8b23864fc000 x1862022098958848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766654.008092:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098958848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766654.008100:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766654.008103:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766654.008107:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098958848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766654.008114:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098958976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766654.008148:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766654.008161:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766654/real 1775766654] req@ffff8b23864fed80 x1862022098958976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766654.008174:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098958976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766654.008182:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766654.008185:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766654.008189:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098958976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766654.903606:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766655.927635:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766656.951701:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766657.975682:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766658.999642:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766659.127610:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766659.127625:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766659.127631:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766659.127645:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4120 00000100:00080000:1.0:1775766659.127660:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766659.127669:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766659.127717:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098959104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766659.127751:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766659.127755:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766659.127757:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766659.127761:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4120 00000100:00080000:1.0:1775766659.127765:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766659.127768:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766659.127779:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766659.127795:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098959232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766659.127816:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766659.127820:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766659.127822:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766659.127828:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4120 00000100:00080000:1.0:1775766659.127832:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766659.127839:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766659.127840:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098959104:192.168.202.128@tcp:250:kworker.0 00000008:00100000:1.0:1775766659.127845:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766659.127860:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098959360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766659.127871:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766659.127875:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766659.127878:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766659.127883:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4120 00000100:00080000:1.0:1775766659.127906:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766659.127911:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766659.127917:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766659.127919:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766659.127934:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098959488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766659.127945:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766659.127949:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766659.127951:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766659.127956:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4120 00000100:00080000:1.0:1775766659.127961:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766659.127966:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766659.127971:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766659/real 1775766659] req@ffff8b23864fce00 x1862022098959104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775766659.127973:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766659.127992:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098959104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766659.127998:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098959616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766659.128004:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766659.128013:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766659.128018:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098959104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766659.128025:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098959232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766659.128041:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766659.128052:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766659/real 1775766659] req@ffff8b23864fd180 x1862022098959232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766659.128063:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098959232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766659.128087:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766659.128092:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766659.128096:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098959232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766659.128127:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098959360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766659.128149:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766659.128160:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766659/real 1775766659] req@ffff8b23864fea00 x1862022098959360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766659.128168:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098959360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766659.128174:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766659.128177:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766659.128179:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098959360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766659.128183:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098959488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766659.128197:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766659.128203:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766659/real 1775766659] req@ffff8b23864fca80 x1862022098959488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766659.128210:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098959488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766659.128215:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766659.128216:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766659.128218:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098959488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766659.128222:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098959616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766659.128232:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766659.128240:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766659/real 1775766659] req@ffff8b23864fe680 x1862022098959616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766659.128251:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098959616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766659.128260:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766659.128264:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766659.128267:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098959616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766660.023690:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766661.047665:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766662.071694:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766663.095681:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766664.119644:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766664.247624:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766664.247640:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766664.247647:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766664.247661:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4125 00000100:00080000:0.0:1775766664.247679:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766664.247699:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766664.247752:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098959744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766664.247819:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766664.247824:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766664.247826:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766664.247831:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4125 00000100:00080000:0.0:1775766664.247836:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766664.247841:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766664.247863:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766664.247876:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098959744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766664.247884:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098959872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766664.247900:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766664.247905:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766664.247908:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766664.247914:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4125 00000100:00080000:0.0:1775766664.247919:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766664.247924:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766664.247930:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766664.247953:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098960000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766664.247965:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766664.247970:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766664.247972:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775766664.247977:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766664.247982:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4125 00000100:00080000:0.0:1775766664.247988:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766664.247993:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766664.247999:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766664.248012:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098960128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766664.248041:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766664.248045:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766664.248048:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766664.248053:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4125 00000100:00080000:0.0:1775766664.248058:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766664.248059:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766664/real 1775766664] req@ffff8b2382dace00 x1862022098959744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766664.248066:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766664.248086:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766664.248097:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098959744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766664.248106:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098960256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766664.248108:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766664.248117:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766664.248125:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098959744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766664.248157:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098959872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766664.248177:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766664.248188:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766664/real 1775766664] req@ffff8b2382daed80 x1862022098959872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766664.248196:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098959872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766664.248202:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766664.248204:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766664.248207:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098959872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766664.248210:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098960000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766664.248224:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766664.248233:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766664/real 1775766664] req@ffff8b2382daea00 x1862022098960000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766664.248246:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098960000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766664.248255:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766664.248259:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766664.248262:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098960000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766664.248278:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098960128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766664.248300:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766664.248308:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766664/real 1775766664] req@ffff8b2382dad500 x1862022098960128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766664.248316:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098960128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766664.248322:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766664.248325:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766664.248327:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098960128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766664.248331:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098960256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766664.248343:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766664.248348:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766664/real 1775766664] req@ffff8b2382dae680 x1862022098960256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766664.248355:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098960256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766664.248360:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766664.248362:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766664.248364:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098960256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766665.143710:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766666.167750:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766667.191661:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766668.215667:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766669.239662:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766669.367554:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766669.367566:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766669.367572:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766669.367591:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4130 00000100:00080000:0.0:1775766669.367605:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766669.367612:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766669.367658:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098960384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766669.367686:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766669.367690:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766669.367691:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766669.367694:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4130 00000100:00080000:0.0:1775766669.367697:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766669.367699:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766669.367709:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766669.367720:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098960512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766669.367728:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766669.367731:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766669.367732:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766669.367736:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4130 00000100:00080000:0.0:1775766669.367739:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766669.367743:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766669.367746:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766669.367754:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098960640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766669.367762:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766669.367765:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1775766669.367766:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098960384:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766669.367767:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766669.367772:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4130 00000100:00080000:0.0:1775766669.367777:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766669.367780:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766669.367783:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766669.367791:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098960768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766669.367797:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766669.367800:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766669.367801:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766669.367803:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4130 00000100:00080000:0.0:1775766669.367806:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766669.367808:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766669.367811:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766669.367820:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098960896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766669.367860:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766669.367914:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766669/real 1775766669] req@ffff8b2382dac380 x1862022098960384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766669.367938:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098960384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766669.367954:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766669.367957:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766669.367961:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098960384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766669.367982:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098960512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766669.368001:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766669.368011:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766669/real 1775766669] req@ffff8b2382dac000 x1862022098960512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766669.368019:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098960512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766669.368025:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766669.368028:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766669.368030:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098960512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766669.368056:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098960640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766669.368067:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766669.368073:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766669/real 1775766669] req@ffff8b2382daf100 x1862022098960640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766669.368081:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098960640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766669.368087:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766669.368088:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766669.368090:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098960640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766669.368095:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098960768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766669.368106:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766669.368112:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766669/real 1775766669] req@ffff8b2382daca80 x1862022098960768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766669.368119:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098960768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766669.368125:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766669.368127:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766669.368128:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098960768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766669.368132:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098960896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766669.368142:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766669.368146:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766669/real 1775766669] req@ffff8b2382dad880 x1862022098960896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766669.368153:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098960896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766669.368158:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766669.368159:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766669.368161:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098960896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766670.263673:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766671.287744:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766672.311648:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766673.335627:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766674.359712:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766674.487562:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766674.487570:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766674.487574:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766674.487582:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4135 00000100:00080000:1.0:1775766674.487592:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766674.487597:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766674.487631:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098961024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766674.487653:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766674.487656:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766674.487658:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766674.487660:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4135 00000100:00080000:1.0:1775766674.487662:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766674.487665:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766674.487673:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766674.487681:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098961152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766674.487689:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766674.487692:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766674.487693:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766674.487695:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4135 00000100:00080000:1.0:1775766674.487697:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766674.487700:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766674.487703:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766674.487711:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098961280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766674.487718:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766674.487722:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766674.487723:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766674.487724:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098961024:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766674.487728:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4135 00000100:00080000:1.0:1775766674.487731:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766674.487736:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766674.487740:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766674.487747:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098961408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766674.487753:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766674.487756:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766674.487757:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766674.487760:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4135 00000100:00080000:1.0:1775766674.487762:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766674.487764:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766674.487767:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766674.487776:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098961536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766674.487791:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766674.487823:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766674/real 1775766674] req@ffff8b23864fdc00 x1862022098961024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766674.487847:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098961024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766674.487855:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766674.487861:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766674.487864:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098961024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766674.487873:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098961152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766674.487887:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766674.487895:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766674/real 1775766674] req@ffff8b23864fd500 x1862022098961152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766674.487904:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098961152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766674.487942:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766674.487946:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766674.487950:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098961152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766674.487975:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098961280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766674.487995:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766674.488003:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766674/real 1775766674] req@ffff8b23864ff100 x1862022098961280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766674.488015:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098961280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766674.488021:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766674.488023:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766674.488025:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098961280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766674.488029:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098961408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766674.488041:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766674.488046:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766674/real 1775766674] req@ffff8b23864fe300 x1862022098961408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766674.488053:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098961408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766674.488058:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766674.488059:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766674.488061:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098961408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766674.488064:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098961536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766674.488074:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766674.488079:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766674/real 1775766674] req@ffff8b23864ffb80 x1862022098961536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766674.488085:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098961536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766674.488094:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766674.488096:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766674.488098:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098961536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775766675.383732:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766676.407661:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766677.431723:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766678.455622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766679.479634:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766679.607643:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766679.607657:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766679.607664:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766679.607679:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4141 00000100:00080000:2.0:1775766679.607696:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766679.607707:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766679.607763:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098961664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766679.607812:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766679.607828:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766679.607831:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766679.607838:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4141 00000100:00080000:2.0:1775766679.607846:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766679.607854:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766679.607894:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766679.607897:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098961664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775766679.607942:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098961792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766679.607969:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766679.607976:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766679.607980:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766679.607988:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4141 00000100:00080000:2.0:1775766679.607997:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766679.608005:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766679.608015:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766679.608016:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775766679.608044:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098961920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766679.608078:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766679.608085:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766679.608088:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766679.608097:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4141 00000100:00100000:3.0:1775766679.608103:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766679/real 1775766679] req@ffff8b23b9ea4a80 x1862022098961664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766679.608106:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766679.608121:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766679.608134:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766679.608141:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022098961664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766679.608155:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1775766679.608156:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098962048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766679.608167:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766679.608180:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098961664:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775766679.608194:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766679.608201:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766679.608204:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766679.608212:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4141 00000100:00080000:2.0:1775766679.608218:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766679.608221:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098961792:192.168.202.128@tcp:38:kworker.0 00000100:00080000:2.0:1775766679.608232:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766679.608241:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766679.608263:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098962176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766679.608273:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766679.608293:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766679/real 1775766679] req@ffff8b23b9ea7100 x1862022098961792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766679.608337:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022098961792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766679.608356:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766679.608361:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766679.608365:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098961792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766679.608372:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098961920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766679.608393:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766679.608403:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766679/real 1775766679] req@ffff8b23b9ea4380 x1862022098961920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766679.608416:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022098961920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766679.608424:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766679.608427:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766679.608431:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098961920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766679.608437:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098962048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766679.608509:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766679.608529:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766679/real 1775766679] req@ffff8b23b9ea4e00 x1862022098962048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766679.608551:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022098962048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766679.608567:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766679.608573:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766679.608579:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098962048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766679.608605:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098962176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766679.608633:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766679.608645:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766679/real 1775766679] req@ffff8b23b9ea7480 x1862022098962176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766679.608665:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022098962176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766679.608679:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766679.608684:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766679.608689:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098962176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766680.503688:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766681.527718:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766682.551660:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766683.575730:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766684.599593:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766684.727572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766684.727585:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766684.727589:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766684.727601:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4146 00000100:00080000:2.0:1775766684.727614:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766684.727619:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766684.727656:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022098962304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766684.727682:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766684.727685:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766684.727687:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766684.727690:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4146 00000100:00080000:2.0:1775766684.727692:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766684.727695:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766684.727704:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766684.727713:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098962432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766684.727722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766684.727725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766684.727726:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766684.727729:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4146 00000100:00080000:2.0:1775766684.727731:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766684.727733:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766684.727737:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766684.727748:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098962560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766684.727759:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766684.727773:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098962304:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775766684.727774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766684.727777:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766684.727788:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4146 00000100:00080000:2.0:1775766684.727793:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766684.727798:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766684.727821:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766684.727836:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098962688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766684.727844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766684.727847:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766684.727849:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766684.727852:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4146 00000100:00080000:2.0:1775766684.727856:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766684.727862:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775766684.727862:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:2.0:1775766684.727869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766684.727879:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098962816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766684.727911:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766684/real 1775766684] req@ffff8b23b9ea6300 x1862022098962304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766684.727930:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022098962304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766684.727940:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766684.727944:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766684.727947:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098962304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766684.727953:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098962432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766684.727970:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766684.727978:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766684/real 1775766684] req@ffff8b23b9ea7480 x1862022098962432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766684.727987:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022098962432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766684.728010:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766684.728013:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766684.728016:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098962432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766684.728022:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098962560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766684.728038:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766684.728046:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766684/real 1775766684] req@ffff8b23b9ea4e00 x1862022098962560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766684.728056:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022098962560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766684.728064:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766684.728066:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766684.728069:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098962560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766684.728094:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098962688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766684.728109:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766684.728117:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766684/real 1775766684] req@ffff8b23b9ea4380 x1862022098962688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766684.728155:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022098962688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766684.728164:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766684.728168:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766684.728172:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098962688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766684.728179:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098962816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766684.728201:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766684.728213:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766684/real 1775766684] req@ffff8b23b9ea7100 x1862022098962816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766684.728226:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022098962816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766684.728232:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766684.728234:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766684.728236:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098962816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766685.623703:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766686.647598:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766687.671615:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766688.695706:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766689.719675:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766689.847604:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766689.847616:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766689.847622:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766689.847634:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4151 00000100:00080000:3.0:1775766689.847647:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766689.847656:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766689.847718:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098962944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766689.847799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766689.847803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766689.847805:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766689.847810:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4151 00000100:00080000:3.0:1775766689.847813:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766689.847817:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766689.847829:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766689.847841:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098963072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766689.847853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766689.847856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766689.847859:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766689.847862:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4151 00000100:00080000:3.0:1775766689.847865:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766689.847868:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766689.847872:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766689.847880:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098962944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766689.847884:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098963200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766689.847896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766689.847899:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766689.847901:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766689.847904:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4151 00000100:00080000:3.0:1775766689.847909:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766689.847912:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766689.847917:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766689.847929:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098963328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766689.847938:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766689.847941:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766689.847942:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766689.847945:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4151 00000100:00080000:3.0:1775766689.847948:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766689.847951:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766689.847955:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766689.847963:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098963456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766689.847968:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766689.848022:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766689/real 1775766689] req@ffff8b23864f2a00 x1862022098962944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766689.848039:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098962944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766689.848050:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766689.848054:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766689.848058:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098962944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766689.848065:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098963072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766689.848083:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766689.848094:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766689/real 1775766689] req@ffff8b23864f3480 x1862022098963072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766689.848104:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098963072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766689.848110:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766689.848113:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766689.848116:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098963072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766689.848140:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098963200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766689.848153:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766689.848161:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766689/real 1775766689] req@ffff8b23864f2d80 x1862022098963200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766689.848170:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098963200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766689.848194:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766689.848197:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766689.848199:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098963200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766689.848204:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098963328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766689.848225:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766689.848236:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766689/real 1775766689] req@ffff8b23864f0000 x1862022098963328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766689.848250:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098963328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766689.848257:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766689.848260:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766689.848269:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098963328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766689.848273:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098963456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766689.848288:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766689.848295:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766689/real 1775766689] req@ffff8b23864f3100 x1862022098963456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766689.848305:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098963456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766689.848317:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766689.848320:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766689.848323:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098963456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766690.743711:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766691.767807:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766692.791695:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766693.815657:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766694.839662:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766694.967569:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766694.967580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766694.967586:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766694.967597:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4156 00000100:00080000:2.0:1775766694.967609:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766694.967616:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766694.967658:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098963584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766694.967686:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766694.967689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766694.967690:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766694.967693:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4156 00000100:00080000:2.0:1775766694.967710:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766694.967715:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766694.967724:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766694.967736:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022098963712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766694.967745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766694.967748:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766694.967750:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766694.967755:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4156 00000100:00100000:0.0:1775766694.967756:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098963584:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775766694.967758:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766694.967764:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766694.967767:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766694.967776:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098963840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766694.967787:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766694.967790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766694.967792:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766694.967795:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4156 00000100:00080000:2.0:1775766694.967797:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766694.967799:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766694.967802:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766694.967809:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098963968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766694.967815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766694.967819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766694.967821:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766694.967825:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4156 00000100:00080000:2.0:1775766694.967830:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775766694.967834:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775766694.967838:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766694.967844:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766694.967859:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098964096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766694.967884:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766694/real 1775766694] req@ffff8b23b9ea4a80 x1862022098963584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766694.967901:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022098963584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766694.967909:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766694.967912:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766694.967916:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098963584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766694.967939:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098963712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766694.967953:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766694.967961:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766694/real 1775766694] req@ffff8b23b9ea5880 x1862022098963712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766694.967969:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022098963712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766694.967975:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766694.967978:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766694.967980:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098963712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766694.967984:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098963840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766694.967997:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766694.968003:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766694/real 1775766694] req@ffff8b23b9ea4000 x1862022098963840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766694.968010:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022098963840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766694.968015:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766694.968017:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766694.968019:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098963840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766694.968022:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098963968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766694.968033:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766694.968038:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766694/real 1775766694] req@ffff8b23b9ea6a00 x1862022098963968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766694.968046:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022098963968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766694.968050:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766694.968052:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766694.968054:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098963968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766694.968057:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098964096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766694.968069:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766694.968075:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766694/real 1775766694] req@ffff8b23b9ea4700 x1862022098964096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766694.968082:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022098964096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766694.968087:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766694.968089:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766694.968090:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098964096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766695.863668:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766696.887745:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766697.911807:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766698.935764:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766699.959767:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766700.087611:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766700.087624:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766700.087631:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766700.087683:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4161 00000100:00080000:2.0:1775766700.087705:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766700.087716:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766700.087782:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098964224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766700.087824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766700.087829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766700.087832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766700.087836:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4161 00000100:00080000:2.0:1775766700.087840:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766700.087844:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766700.087856:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766700.087870:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098964352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766700.087883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766700.087887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766700.087889:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766700.087895:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4161 00000100:00080000:2.0:1775766700.087899:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766700.087905:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766700.087913:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766700.087918:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098964224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775766700.087936:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022098964480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766700.087953:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766700.087960:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766700.087963:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766700.087971:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4161 00000100:00080000:2.0:1775766700.087977:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766700.087983:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766700.087991:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766700.088011:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775766700.088012:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098964608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766700.088026:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766700.088030:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766700.088032:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766700.088037:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4161 00000100:00080000:2.0:1775766700.088042:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766700.088045:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766700.088050:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766700.088062:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098964736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766700.088077:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766700/real 1775766700] req@ffff8b23b9ea5500 x1862022098964224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766700.088100:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022098964224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766700.088113:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766700.088118:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766700.088123:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098964224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766700.088131:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098964352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766700.088152:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766700.088164:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766700/real 1775766700] req@ffff8b23b9ea5c00 x1862022098964352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766700.088176:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022098964352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766700.088183:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766700.088186:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766700.088190:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098964352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766700.088216:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098964480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766700.088230:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766700.088239:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766700/real 1775766700] req@ffff8b23b9ea6d80 x1862022098964480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766700.088249:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022098964480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766700.088257:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766700.088260:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766700.088263:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098964480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766700.088268:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098964608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766700.088282:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766700.088289:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766700/real 1775766700] req@ffff8b23b9ea6680 x1862022098964608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766700.088299:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022098964608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766700.088307:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766700.088310:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766700.088313:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098964608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766700.088318:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098964736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766700.088334:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766700.088341:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766700/real 1775766700] req@ffff8b23b9ea5f80 x1862022098964736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766700.088350:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022098964736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766700.088358:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766700.088361:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766700.088363:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098964736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766700.983632:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766702.007675:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766703.031622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766704.055618:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766705.079688:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766705.207584:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766705.207610:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766705.207616:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766705.207626:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4166 00000100:00080000:1.0:1775766705.207639:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766705.207646:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766705.207693:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098964864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766705.207731:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766705.207735:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766705.207737:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766705.207742:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4166 00000100:00080000:1.0:1775766705.207747:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766705.207752:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766705.207765:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766705.207782:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098964992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766705.207792:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098964864:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766705.207797:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766705.207801:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766705.207803:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766705.207807:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4166 00000100:00080000:1.0:1775766705.207811:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766705.207814:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766705.207819:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766705.207831:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098965120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766705.207838:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766705.207842:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766705.207844:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766705.207848:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4166 00000100:00080000:1.0:1775766705.207889:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766705.207892:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766705.207896:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766705.207901:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766705.207907:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098965248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766705.207916:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766705.207918:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766705.207919:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766705.207922:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4166 00000100:00080000:1.0:1775766705.207924:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766705.207926:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766705.207929:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766705.207940:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098965376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766705.207949:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766705/real 1775766705] req@ffff8b23864ffb80 x1862022098964864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766705.207968:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098964864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766705.207977:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766705.207981:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766705.207985:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098964864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766705.208008:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098964992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766705.208023:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766705.208032:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766705/real 1775766705] req@ffff8b23864fe300 x1862022098964992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766705.208040:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098964992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766705.208045:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766705.208048:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766705.208050:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098964992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766705.208053:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098965120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766705.208064:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766705.208069:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766705/real 1775766705] req@ffff8b23864ff100 x1862022098965120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766705.208076:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098965120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766705.208081:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766705.208083:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766705.208084:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098965120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766705.208088:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098965248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766705.208100:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766705.208105:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766705/real 1775766705] req@ffff8b23864fd500 x1862022098965248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766705.208128:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098965248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766705.208134:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766705.208136:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766705.208138:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098965248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766705.208141:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098965376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766705.208155:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766705.208161:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766705/real 1775766705] req@ffff8b23864fdc00 x1862022098965376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766705.208168:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098965376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766705.208174:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766705.208176:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766705.208178:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098965376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766706.103616:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766707.127690:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766708.151642:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766709.175744:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766710.199723:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766710.327567:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766710.327585:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766710.327593:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766710.327608:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4171 00000100:00080000:1.0:1775766710.327626:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766710.327634:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766710.327690:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098965504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766710.327728:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766710.327732:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766710.327734:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766710.327749:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4171 00000100:00080000:1.0:1775766710.327752:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766710.327755:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766710.327767:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766710.327778:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098965632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766710.327789:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766710.327794:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766710.327795:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766710.327800:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4171 00000100:00080000:1.0:1775766710.327803:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766710.327807:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766710.327811:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766710.327821:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098965760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766710.327831:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766710.327834:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766710.327836:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766710.327849:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098965504:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766710.327852:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4171 00000100:00080000:1.0:1775766710.327863:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766710.327868:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766710.327873:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766710.327886:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098965888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766710.327896:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766710.327899:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766710.327901:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766710.327905:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4171 00000100:00080000:1.0:1775766710.327908:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766710.327912:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766710.327918:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766710.327928:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098966016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766710.327951:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766710.328001:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766710/real 1775766710] req@ffff8b23864fe680 x1862022098965504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766710.328025:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098965504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766710.328038:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766710.328044:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766710.328049:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098965504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766710.328057:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098965632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766710.328079:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766710.328090:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766710/real 1775766710] req@ffff8b23864fca80 x1862022098965632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766710.328099:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098965632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766710.328106:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766710.328108:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766710.328127:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098965632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766710.328151:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098965760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766710.328168:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766710.328176:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766710/real 1775766710] req@ffff8b23864fea00 x1862022098965760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766710.328186:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098965760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766710.328198:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766710.328201:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766710.328204:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098965760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766710.328209:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098965888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766710.328222:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766710.328245:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766710/real 1775766710] req@ffff8b23864fd180 x1862022098965888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766710.328262:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098965888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766710.328275:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766710.328280:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766710.328285:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098965888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766710.328310:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098966016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766710.328340:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766710.328356:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766710/real 1775766710] req@ffff8b23864fce00 x1862022098966016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766710.328372:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098966016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766710.328379:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766710.328382:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766710.328386:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098966016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766711.223791:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766712.247718:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766713.271757:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766714.295686:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766715.319719:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766715.447575:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766715.447583:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766715.447587:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766715.447595:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4176 00000100:00080000:1.0:1775766715.447605:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766715.447611:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766715.447651:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098966144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766715.447678:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766715.447681:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766715.447683:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766715.447686:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4176 00000100:00080000:1.0:1775766715.447689:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766715.447691:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766715.447699:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766715.447709:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098966272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766715.447717:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766715.447736:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766715.447738:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766715.447743:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4176 00000100:00080000:1.0:1775766715.447749:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766715.447750:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098966144:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766715.447755:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766715.447763:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766715.447775:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098966400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766715.447785:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766715.447790:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766715.447793:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766715.447799:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4176 00000100:00080000:1.0:1775766715.447817:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766715.447823:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766715.447830:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766715.447867:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098966528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766715.447875:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098966272:192.168.202.128@tcp:38:kworker.0 00000100:00080000:1.0:1775766715.447902:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766715.447905:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766715.447907:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766715.447912:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4176 00000100:00080000:1.0:1775766715.447916:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766715.447921:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766715.447927:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766715.447944:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098966656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766715.447947:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098966400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766715.447990:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098966528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766715.448013:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098966656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766716.343652:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766717.367688:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766718.391710:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775766718.519748:0:2363:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:3.0:1775766718.519795:0:2363:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775766718.519813:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775766718.519856:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775766718.519871:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775766718.519884:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775766718.519896:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775766718.519909:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766715/real 1775766718] req@ffff8b23864fdf80 x1862022098966400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766718.519942:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098966400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766718.519956:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766718.519967:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766718.519973:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098966400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766718.519983:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766715/real 1775766718] req@ffff8b23864ff800 x1862022098966528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766718.520002:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098966528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766718.520021:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766718.520025:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766718.520029:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098966528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766718.520037:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766715/real 1775766718] req@ffff8b23864fed80 x1862022098966144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766718.520050:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098966144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766718.520058:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766718.520062:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766718.520065:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098966144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766718.520072:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766715/real 1775766718] req@ffff8b23864fc000 x1862022098966272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766718.520085:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098966272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766718.520094:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766718.520098:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766718.520115:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098966272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766718.520165:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766715/real 1775766718] req@ffff8b23864fc380 x1862022098966656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766718.520179:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098966656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766718.520188:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766718.520192:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766718.520196:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098966656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766719.415725:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766720.439708:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766720.567558:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766720.567566:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766720.567570:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766720.567580:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4181 00000100:00080000:2.0:1775766720.567589:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766720.567595:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766720.567627:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022098966784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766720.567651:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766720.567654:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766720.567655:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766720.567658:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4181 00000100:00080000:2.0:1775766720.567660:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766720.567662:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766720.567670:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766720.567678:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022098966912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766720.567687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766720.567701:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766720.567703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766720.567706:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4181 00000100:00080000:2.0:1775766720.567708:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766720.567710:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766720.567713:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766720.567719:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022098967040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766720.567725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766720.567728:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766720.567729:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766720.567732:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4181 00000100:00080000:2.0:1775766720.567734:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766720.567737:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766720.567740:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766720.567746:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022098967168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766720.567755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766720.567757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766720.567759:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766720.567762:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4181 00000100:00080000:2.0:1775766720.567764:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766720.567769:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766720.567776:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766720.567796:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098966784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775766720.567796:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022098967296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766720.567946:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766720.568033:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766720/real 1775766720] req@ffff8b23b9ea7b80 x1862022098966784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766720.568066:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022098966784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766720.568083:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766720.568107:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766720.568113:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098966784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766720.568126:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098966912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766720.568171:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766720.568210:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766720/real 1775766720] req@ffff8b23b9ea5f80 x1862022098966912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766720.568230:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022098966912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766720.568241:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766720.568246:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766720.568252:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098966912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766720.568261:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098967040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766720.568296:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766720.568312:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766720/real 1775766720] req@ffff8b23b9ea6680 x1862022098967040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766720.568326:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022098967040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766720.568337:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766720.568341:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766720.568345:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098967040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766720.568393:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098967168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766720.568423:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766720.568568:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766720/real 1775766720] req@ffff8b23b9ea6d80 x1862022098967168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766720.568582:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022098967168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766720.568589:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766720.568594:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766720.568597:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098967168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766720.568604:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098967296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766720.568626:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766720.568639:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766720/real 1775766720] req@ffff8b23b9ea5c00 x1862022098967296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766720.568653:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022098967296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766720.568662:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766720.568666:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766720.568669:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098967296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766721.463633:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766722.487645:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766723.511588:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766724.535615:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766725.559610:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766725.687591:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766725.687603:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766725.687608:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766725.687620:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4187 00000100:00080000:3.0:1775766725.687632:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766725.687640:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766725.687685:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098967424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766725.687720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766725.687724:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766725.687726:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766725.687730:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4187 00000100:00080000:3.0:1775766725.687734:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766725.687737:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766725.687747:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766725.687759:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098967552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766725.687772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766725.687776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766725.687779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766725.687783:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4187 00000100:00080000:3.0:1775766725.687788:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766725.687794:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098967424:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766725.687795:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766725.687806:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766725.687821:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098967680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766725.687849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766725.687854:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766725.687857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766725.687863:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4187 00000100:00080000:3.0:1775766725.687870:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766725.687876:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766725.687885:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766725.687907:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766725.687908:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098967808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766725.687927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766725.687931:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766725.687933:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766725.687938:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4187 00000100:00080000:3.0:1775766725.687953:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766725.687957:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766725.687964:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766725.687983:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766725/real 1775766725] req@ffff8b23864f0700 x1862022098967424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766725.687984:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098967936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766725.688044:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098967424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766725.688068:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766725.688073:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766725.688078:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098967424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766725.688086:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098967552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766725.688113:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766725.688125:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766725/real 1775766725] req@ffff8b23864f1180 x1862022098967552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766725.688145:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098967552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766725.688152:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766725.688155:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766725.688159:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098967552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766725.688182:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098967680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766725.688199:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766725.688208:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766725/real 1775766725] req@ffff8b23864f3b80 x1862022098967680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766725.688220:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098967680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766725.688228:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766725.688231:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766725.688233:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098967680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766725.688239:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098967808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766725.688254:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766725.688262:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766725/real 1775766725] req@ffff8b23864f1c00 x1862022098967808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766725.688273:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098967808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766725.688293:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766725.688298:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766725.688303:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098967808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766725.688324:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098967936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766725.688358:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766725.688398:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766725/real 1775766725] req@ffff8b23864f1f80 x1862022098967936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766725.688413:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098967936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766725.688422:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766725.688426:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766725.688429:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098967936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766726.583647:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766727.607660:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766728.631610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766729.655723:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766730.679611:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766730.807571:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766730.807579:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766730.807583:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766730.807592:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4192 00000100:00080000:0.0:1775766730.807601:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766730.807606:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766730.807642:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098968064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766730.807666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766730.807669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766730.807670:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766730.807673:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4192 00000100:00080000:0.0:1775766730.807676:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766730.807679:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766730.807686:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766730.807695:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098968192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766730.807704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766730.807705:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098968064:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766730.807711:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766730.807713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766730.807717:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4192 00000100:00080000:0.0:1775766730.807720:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766730.807722:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766730.807726:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766730.807735:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098968320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766730.807749:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766730.807752:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766730.807753:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766730.807756:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4192 00000100:00080000:0.0:1775766730.807761:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766730.807764:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766730.807768:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766730.807774:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766730.807783:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098968448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766730.807793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766730.807797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766730.807799:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766730.807803:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4192 00000100:00080000:0.0:1775766730.807808:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766730.807812:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766730.807818:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766730.807819:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766730/real 1775766730] req@ffff8b2382dad880 x1862022098968064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766730.807837:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098968576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766730.807845:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098968064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766730.807854:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766730.807858:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766730.807863:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098968064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766730.807885:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098968192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766730.807903:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766730.807911:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766730/real 1775766730] req@ffff8b2382daca80 x1862022098968192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766730.807919:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098968192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766730.807925:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766730.807927:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766730.807929:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098968192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766730.807933:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098968320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766730.807943:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766730.807949:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766730/real 1775766730] req@ffff8b2382daf100 x1862022098968320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766730.807956:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098968320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766730.807965:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766730.807967:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766730.807969:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098968320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766730.807972:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098968448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766730.807982:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766730.807987:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766730/real 1775766730] req@ffff8b2382dac000 x1862022098968448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766730.807994:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098968448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766730.807999:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766730.808000:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766730.808003:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098968448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766730.808008:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098968576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766730.808025:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766730.808035:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766730/real 1775766730] req@ffff8b2382dac380 x1862022098968576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766730.808047:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098968576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766730.808056:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766730.808059:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766730.808063:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098968576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766731.703634:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766732.727591:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766733.751811:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766734.775639:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766735.799591:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766735.927614:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766735.927627:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766735.927633:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766735.927647:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4197 00000100:00080000:0.0:1775766735.927663:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766735.927672:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766735.927747:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098968704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766735.927788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766735.927792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766735.927795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766735.927799:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4197 00000100:00080000:0.0:1775766735.927804:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766735.927812:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766735.927825:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766735.927841:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098968832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766735.927849:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098968704:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766735.927881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766735.927886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766735.927888:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766735.927895:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4197 00000100:00080000:0.0:1775766735.927899:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766735.927904:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766735.927908:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766735.927924:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098968960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766735.927935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766735.927939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766735.927941:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766735.927947:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4197 00000800:00000100:3.0:1775766735.927959:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766735.927961:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766735.927968:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766735.927976:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766735.927993:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098969088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766735.928004:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766735.928007:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766735.928011:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766735.928014:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766735/real 1775766735] req@ffff8b2382dae680 x1862022098968704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766735.928018:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4197 00000100:00080000:0.0:1775766735.928025:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766735.928033:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766735.928040:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766735.928059:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098969216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766735.928065:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098968704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766735.928122:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766735.928127:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766735.928129:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098968704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766735.928158:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098968832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766735.928186:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766735.928196:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766735/real 1775766735] req@ffff8b2382dad500 x1862022098968832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766735.928205:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098968832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766735.928211:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766735.928213:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766735.928215:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098968832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766735.928220:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098968960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766735.928231:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766735.928238:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766735/real 1775766735] req@ffff8b2382daea00 x1862022098968960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766735.928245:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098968960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766735.928250:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766735.928252:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766735.928254:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098968960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766735.928258:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098969088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766735.928293:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766735.928308:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766735/real 1775766735] req@ffff8b2382daed80 x1862022098969088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766735.928316:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098969088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766735.928321:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766735.928323:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766735.928325:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098969088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766735.928330:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098969216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766735.928343:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766735.928348:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766735/real 1775766735] req@ffff8b2382dace00 x1862022098969216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766735.928355:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098969216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766735.928361:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766735.928363:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766735.928365:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098969216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766736.823703:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766737.847641:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766738.871640:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766739.895653:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766740.919636:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766741.047572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766741.047584:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766741.047589:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766741.047600:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4202 00000100:00080000:0.0:1775766741.047615:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766741.047622:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766741.047677:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098969344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766741.047701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766741.047704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766741.047706:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766741.047709:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4202 00000100:00080000:0.0:1775766741.047713:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766741.047716:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766741.047725:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766741.047738:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098969472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766741.047747:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098969344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766741.047758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766741.047766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766741.047771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766741.047779:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4202 00000100:00080000:0.0:1775766741.047783:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766741.047789:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766741.047796:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766741.047813:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098969600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766741.047824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766741.047828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766741.047830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766741.047836:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4202 00000100:00080000:0.0:1775766741.047841:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766741.047850:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775766741.047853:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:0.0:1775766741.047860:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766741.047874:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098969728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766741.047882:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766741.047885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766741.047888:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766741.047892:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4202 00000100:00080000:0.0:1775766741.047894:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766741.047897:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766741.047900:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766741.047906:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766741/real 1775766741] req@ffff8b2382daf480 x1862022098969344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766741.047917:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098969856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766741.047956:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098969344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766741.047966:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766741.047969:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766741.047973:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098969344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766741.047998:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098969472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766741.048028:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766741.048042:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766741/real 1775766741] req@ffff8b2382dafb80 x1862022098969472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766741.048056:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098969472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766741.048064:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766741.048066:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766741.048068:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098969472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766741.048074:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098969600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766741.048095:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766741.048106:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766741/real 1775766741] req@ffff8b2382dac700 x1862022098969600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766741.048136:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098969600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766741.048146:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766741.048151:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766741.048155:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098969600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766741.048163:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098969728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766741.048198:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766741.048206:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766741/real 1775766741] req@ffff8b2382dadf80 x1862022098969728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766741.048233:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098969728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766741.048240:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766741.048242:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766741.048244:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098969728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766741.048249:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098969856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766741.048265:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766741.048272:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766741/real 1775766741] req@ffff8b2382dace00 x1862022098969856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766741.048280:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098969856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766741.048285:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766741.048287:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766741.048289:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098969856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766741.943632:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766742.967579:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766743.991571:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766745.015628:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766746.039611:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766746.167524:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766746.167533:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766746.167537:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766746.167546:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4207 00000100:00080000:0.0:1775766746.167557:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766746.167563:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766746.167617:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098969984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766746.167651:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766746.167654:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766746.167655:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766746.167659:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4207 00000100:00080000:0.0:1775766746.167661:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766746.167664:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766746.167673:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766746.167683:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098970112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766746.167691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766746.167694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766746.167696:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766746.167699:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4207 00000100:00080000:0.0:1775766746.167701:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766746.167704:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766746.167708:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766746.167717:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098970240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766746.167724:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766746.167726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766746.167728:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766746.167732:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4207 00000100:00100000:3.0:1775766746.167741:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098969984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766746.167756:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766746.167760:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766746.167765:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766746.167776:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098970368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766746.167786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766746.167798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766746.167800:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766746.167810:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4207 00000100:00080000:0.0:1775766746.167814:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766746.167820:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775766746.167821:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:0.0:1775766746.167827:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766746.167842:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098970496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766746.167869:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766746/real 1775766746] req@ffff8b2382daed80 x1862022098969984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766746.167898:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098969984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766746.167907:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766746.167910:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766746.167914:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098969984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766746.167919:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098970112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766746.167935:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766746.167943:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766746/real 1775766746] req@ffff8b2382daea00 x1862022098970112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766746.167952:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098970112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766746.167963:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766746.167966:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766746.167969:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098970112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766746.167973:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098970240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766746.167984:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766746.167990:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766746/real 1775766746] req@ffff8b2382dad500 x1862022098970240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766746.167998:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098970240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766746.168004:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766746.168006:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766746.168008:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098970240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766746.168034:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098970368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766746.168047:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766746.168054:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766746/real 1775766746] req@ffff8b2382dae680 x1862022098970368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766746.168063:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098970368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766746.168068:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766746.168070:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766746.168073:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098970368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766746.168077:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098970496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766746.168099:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766746.168105:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766746/real 1775766746] req@ffff8b2382dac380 x1862022098970496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766746.168113:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098970496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766746.168119:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766746.168121:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766746.168123:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098970496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766747.063638:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766748.087656:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766749.111723:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766750.135640:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766751.159603:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766751.287677:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766751.287687:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766751.287692:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766751.287702:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4212 00000100:00080000:1.0:1775766751.287714:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766751.287720:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766751.287762:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098970624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766751.287807:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766751.287811:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766751.287813:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766751.287816:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4212 00000100:00080000:1.0:1775766751.287819:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766751.287822:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766751.287831:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766751.287843:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098970752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766751.287854:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766751.287857:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766751.287860:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766751.287864:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4212 00000100:00080000:1.0:1775766751.287869:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766751.287872:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766751.287877:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766751.287886:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098970880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766751.287904:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766751.287904:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098970624:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766751.287916:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766751.287919:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766751.287924:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4212 00000100:00080000:1.0:1775766751.287927:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766751.287931:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766751.287936:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766751.287948:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098971008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766751.287957:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766751.287961:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766751.287963:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766751.287966:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4212 00000100:00080000:1.0:1775766751.287971:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766751.287974:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766751.287978:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766751.287987:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098971136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766751.288006:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766751.288062:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766751/real 1775766751] req@ffff8b23864fc380 x1862022098970624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766751.288080:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098970624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766751.288091:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766751.288094:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766751.288098:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098970624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766751.288103:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098970752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766751.288159:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766751.288175:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766751/real 1775766751] req@ffff8b23864fc000 x1862022098970752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766751.288188:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098970752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766751.288196:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766751.288198:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766751.288201:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098970752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766751.288228:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098970880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766751.288244:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766751.288252:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766751/real 1775766751] req@ffff8b23864fed80 x1862022098970880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766751.288260:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098970880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766751.288265:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766751.288268:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766751.288270:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098970880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766751.288274:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098971008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766751.288285:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766751.288290:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766751/real 1775766751] req@ffff8b23864ff800 x1862022098971008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766751.288297:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098971008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766751.288303:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766751.288305:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766751.288307:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098971008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766751.288311:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098971136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766751.288320:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766751.288326:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766751/real 1775766751] req@ffff8b23864fdf80 x1862022098971136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766751.288332:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098971136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766751.288337:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766751.288338:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766751.288340:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098971136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766752.183603:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766753.207674:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766754.231683:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766755.255708:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766756.279690:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766756.407618:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766756.407627:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766756.407631:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766756.407640:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4217 00000100:00080000:0.0:1775766756.407652:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766756.407658:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766756.407699:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098971264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766756.407737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766756.407751:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766756.407754:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766756.407763:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4217 00000100:00080000:0.0:1775766756.407769:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766756.407774:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766756.407785:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766756.407819:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098971392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766756.407832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766756.407835:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766756.407838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766756.407843:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4217 00000100:00100000:3.0:1775766756.407847:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098971264:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766756.407848:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766756.407856:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766756.407862:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766756.407874:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098971520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766756.407883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766756.407886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766756.407888:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766756.407892:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4217 00000100:00080000:0.0:1775766756.407912:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766756.407915:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766756.407919:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766756.407929:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098971648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766756.407937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1775766756.407943:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766756.407944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766756.407949:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766756.407953:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4217 00000100:00080000:0.0:1775766756.407957:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766756.407978:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766756.407983:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766756.408001:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098971776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766756.408012:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766756/real 1775766756] req@ffff8b2382dac000 x1862022098971264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766756.408049:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098971264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766756.408094:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766756.408117:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766756.408122:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098971264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766756.408132:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098971392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766756.408159:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766756.408171:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766756/real 1775766756] req@ffff8b2382daf100 x1862022098971392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766756.408182:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098971392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766756.408188:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766756.408191:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766756.408193:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098971392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766756.408215:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098971520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766756.408234:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766756.408246:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766756/real 1775766756] req@ffff8b2382daca80 x1862022098971520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766756.408260:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098971520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766756.408270:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766756.408274:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766756.408279:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098971520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766756.408286:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098971648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766756.408307:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766756.408316:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766756/real 1775766756] req@ffff8b2382dad880 x1862022098971648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766756.408326:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098971648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766756.408332:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766756.408334:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766756.408336:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098971648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766756.408341:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098971776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766756.408352:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766756.408358:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766756/real 1775766756] req@ffff8b2382dac380 x1862022098971776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766756.408366:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098971776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766756.408371:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766756.408373:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766756.408375:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098971776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766757.303689:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766758.327761:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766759.351666:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766760.375658:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766761.399626:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766761.527540:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766761.527550:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766761.527555:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766761.527565:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4222 00000100:00080000:3.0:1775766761.527579:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766761.527587:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766761.527631:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098971904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766761.527658:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766761.527661:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766761.527663:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766761.527666:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4222 00000100:00080000:3.0:1775766761.527668:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766761.527671:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766761.527680:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766761.527690:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098972032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766761.527700:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766761.527702:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098971904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766761.527704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766761.527707:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766761.527715:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4222 00000100:00080000:3.0:1775766761.527718:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766761.527721:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766761.527724:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766761.527733:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098972160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766761.527739:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766761.527742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766761.527743:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766761.527746:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4222 00000100:00080000:3.0:1775766761.527749:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766761.527752:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766761.527755:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766761.527764:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098972288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766761.527769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766761.527771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766761.527773:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766761.527776:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4222 00000100:00080000:3.0:1775766761.527779:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766761.527781:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766761.527783:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766761.527790:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098972416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766761.527801:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766761.527858:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766761/real 1775766761] req@ffff8b23864f0a80 x1862022098971904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766761.527879:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098971904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766761.527890:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766761.527894:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766761.527898:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098971904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766761.527922:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098972032:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766761.527957:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766761.527968:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766761/real 1775766761] req@ffff8b23864f2680 x1862022098972032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766761.527977:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098972032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766761.527983:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766761.527985:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766761.527988:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098972032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766761.527992:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098972160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766761.528021:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766761.528029:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766761/real 1775766761] req@ffff8b23864f0380 x1862022098972160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766761.528039:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098972160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766761.528048:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766761.528052:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766761.528055:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098972160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766761.528063:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098972288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766761.528087:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766761.528099:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766761/real 1775766761] req@ffff8b23864f1f80 x1862022098972288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766761.528112:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098972288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766761.528121:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766761.528124:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766761.528128:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098972288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766761.528135:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098972416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766761.528158:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766761.528170:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766761/real 1775766761] req@ffff8b23864f1c00 x1862022098972416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766761.528183:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098972416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766761.528191:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766761.528195:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766761.528198:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098972416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766762.423623:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766763.447636:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766764.471598:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766765.495612:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766766.519675:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766766.647613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766766.647626:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766766.647631:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766766.647642:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4228 00000100:00080000:3.0:1775766766.647658:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766766.647666:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766766.647714:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098972544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766766.647743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766766.647749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766766.647750:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766766.647754:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4228 00000100:00080000:3.0:1775766766.647756:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766766.647759:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766766.647768:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766766.647779:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098972672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766766.647789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766766.647792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766766.647794:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766766.647798:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4228 00000100:00080000:3.0:1775766766.647800:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766766.647803:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766766.647806:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766766.647815:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098972800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766766.647817:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098972544:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766766.647832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766766.647836:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766766.647837:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766766.647868:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4228 00000100:00080000:3.0:1775766766.647871:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766766.647874:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766766.647880:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766766.647890:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098972928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766766.647899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766766.647903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766766.647904:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766766.647907:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4228 00000100:00080000:3.0:1775766766.647910:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766766.647913:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766766.647916:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766766.647924:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098973056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766766.647944:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766766.648038:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766766/real 1775766766] req@ffff8b23864f3b80 x1862022098972544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766766.648060:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098972544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766766.648070:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766766.648074:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766766.648077:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098972544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766766.648084:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098972672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766766.648105:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766766.648116:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766766/real 1775766766] req@ffff8b23864f1180 x1862022098972672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766766.648124:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098972672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766766.648129:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766766.648132:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766766.648134:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098972672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766766.648157:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098972800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766766.648168:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766766.648174:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766766/real 1775766766] req@ffff8b23864f0700 x1862022098972800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766766.648186:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098972800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766766.648192:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766766.648194:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766766.648196:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098972800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766766.648201:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098972928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766766.648222:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766766.648233:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766766/real 1775766766] req@ffff8b23864f3100 x1862022098972928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766766.648246:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098972928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766766.648254:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766766.648258:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766766.648262:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098972928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766766.648267:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098973056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766766.648284:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766766.648291:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766766/real 1775766766] req@ffff8b23864f0000 x1862022098973056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766766.648299:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098973056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766766.648309:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766766.648311:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766766.648313:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098973056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766767.543672:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766768.567679:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766769.591606:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766770.615617:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766771.639644:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766771.767580:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766771.767611:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766771.767617:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766771.767630:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4233 00000100:00080000:0.0:1775766771.767646:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766771.767669:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766771.767715:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098973184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766771.767762:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766771.767765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766771.767766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766771.767770:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4233 00000100:00080000:0.0:1775766771.767773:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766771.767777:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766771.767802:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766771.767815:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098973312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766771.767815:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098973184:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766771.767839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766771.767841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766771.767843:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766771.767847:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4233 00000100:00080000:0.0:1775766771.767850:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766771.767853:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766771.767856:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766771.767867:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098973440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766771.767874:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766771.767878:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766771.767879:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766771.767882:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4233 00000100:00080000:0.0:1775766771.767885:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766771.767887:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766771.767890:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766771.767932:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098973568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766771.767937:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766771.767947:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766771.767952:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766771.767955:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766771.767960:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4233 00000100:00080000:0.0:1775766771.767966:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766771.767971:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766771.767977:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766771.767988:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766771/real 1775766771] req@ffff8b2382dae680 x1862022098973184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766771.767994:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098973696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766771.768015:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098973184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766771.768024:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766771.768029:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766771.768033:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098973184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766771.768058:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098973312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766771.768077:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766771.768085:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766771/real 1775766771] req@ffff8b2382dad500 x1862022098973312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766771.768094:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098973312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766771.768099:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766771.768101:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766771.768104:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098973312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766771.768108:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098973440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766771.768119:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766771.768125:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766771/real 1775766771] req@ffff8b2382daea00 x1862022098973440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766771.768132:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098973440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766771.768138:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766771.768139:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766771.768141:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098973440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766771.768145:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098973568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766771.768156:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766771.768162:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766771/real 1775766771] req@ffff8b2382daed80 x1862022098973568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766771.768174:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098973568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766771.768178:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766771.768180:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766771.768182:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098973568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766771.768185:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098973696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766771.768195:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766771.768200:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766771/real 1775766771] req@ffff8b2382dace00 x1862022098973696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766771.768206:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098973696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766771.768211:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766771.768213:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766771.768215:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098973696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766772.663662:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766773.687653:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766774.711677:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766775.735665:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766776.759672:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766776.887551:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766776.887562:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766776.887567:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766776.887578:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4238 00000100:00080000:0.0:1775766776.887591:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766776.887598:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766776.887645:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098973824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766776.887674:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766776.887677:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766776.887679:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766776.887682:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4238 00000100:00080000:0.0:1775766776.887684:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766776.887687:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766776.887696:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766776.887706:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098973952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766776.887721:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766776.887723:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766776.887725:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766776.887731:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4238 00000100:00100000:3.0:1775766776.887731:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098973824:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766776.887735:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766776.887742:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766776.887745:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766776.887753:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098974080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766776.887760:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766776.887763:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766776.887764:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766776.887767:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4238 00000100:00080000:0.0:1775766776.887768:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766776.887771:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766776.887774:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766776.887782:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098974208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766776.887787:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766776.887789:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766776.887790:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766776.887792:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4238 00000100:00080000:0.0:1775766776.887796:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766776.887798:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766776.887802:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766776.887813:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098974336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766776.887827:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766776.887886:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766776/real 1775766776] req@ffff8b2382dadf80 x1862022098973824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766776.887901:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098973824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766776.887914:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766776.887918:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766776.887921:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098973824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766776.887946:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098973952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766776.887963:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766776.887972:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766776/real 1775766776] req@ffff8b2382dac700 x1862022098973952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766776.887980:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098973952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766776.887986:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766776.887989:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766776.887991:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098973952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766776.887994:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098974080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766776.888005:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766776.888010:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766776/real 1775766776] req@ffff8b2382dafb80 x1862022098974080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766776.888017:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098974080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766776.888023:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766776.888025:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766776.888026:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098974080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766776.888030:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098974208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766776.888041:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766776.888046:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766776/real 1775766776] req@ffff8b2382daf480 x1862022098974208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766776.888053:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098974208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766776.888059:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766776.888061:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766776.888063:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098974208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766776.888066:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098974336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766776.888075:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766776.888080:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766776/real 1775766776] req@ffff8b2382dace00 x1862022098974336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766776.888087:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098974336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766776.888092:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766776.888093:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766776.888095:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098974336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766777.783677:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766778.807676:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766779.831599:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766780.855672:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766781.879547:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766782.007577:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766782.007584:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766782.007589:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766782.007598:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4243 00000100:00080000:3.0:1775766782.007610:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766782.007615:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766782.007651:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098974464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766782.007678:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766782.007680:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766782.007682:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766782.007687:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4243 00000100:00080000:3.0:1775766782.007691:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766782.007695:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766782.007715:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766782.007721:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098974464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766782.007727:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098974592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766782.007737:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766782.007739:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766782.007741:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766782.007744:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4243 00000100:00080000:3.0:1775766782.007746:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766782.007749:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766782.007753:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766782.007761:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098974720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766782.007771:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766782.007774:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766782.007775:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766782.007778:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4243 00000100:00080000:3.0:1775766782.007780:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766782.007782:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766782.007797:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766782.007810:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098974848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766782.007817:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766782.007821:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766782.007824:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766782.007827:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4243 00000100:00100000:0.0:1775766782.007830:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098974592:192.168.202.128@tcp:38:kworker.0 00000100:00080000:3.0:1775766782.007843:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766782.007847:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766782.007852:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766782.007865:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098974976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766782.007869:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098974720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766782.007895:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098974848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766782.007913:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098974976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766782.903626:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766783.927750:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766784.951716:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775766785.079840:0:2364:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775766785.079891:0:2364:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766785.079912:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766785.079987:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766785.080013:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766785.080031:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766785.080049:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775766785.080066:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766782/real 1775766785] req@ffff8b23864f3480 x1862022098974592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766785.080101:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098974592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766785.080127:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766785.080134:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766785.080139:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098974592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766785.080153:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766782/real 1775766785] req@ffff8b23864f2a00 x1862022098974720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766785.080168:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098974720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766785.080178:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766785.080183:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766785.080187:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098974720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766785.080195:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766782/real 1775766785] req@ffff8b23864f0000 x1862022098974848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766785.080210:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098974848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766785.080219:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766785.080224:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766785.080229:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098974848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766785.080237:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766782/real 1775766785] req@ffff8b23864f2d80 x1862022098974464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766785.080252:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098974464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766785.080261:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766785.080266:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766785.080270:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098974464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766785.080312:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766782/real 1775766785] req@ffff8b23864f3100 x1862022098974976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766785.080328:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098974976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766785.080337:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766785.080343:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766785.080346:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098974976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766785.975665:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766786.999626:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766787.127506:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766787.127517:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766787.127522:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766787.127533:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4248 00000100:00080000:3.0:1775766787.127546:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766787.127553:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766787.127592:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098975104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766787.127620:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766787.127623:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766787.127624:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766787.127627:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4248 00000100:00080000:3.0:1775766787.127630:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766787.127632:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766787.127642:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766787.127658:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098975232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766787.127669:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766787.127674:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775766787.127675:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098975104:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766787.127676:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766787.127685:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4248 00000100:00080000:3.0:1775766787.127689:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766787.127694:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766787.127700:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766787.127715:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098975360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766787.127726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766787.127730:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766787.127748:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766787.127753:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4248 00000100:00080000:3.0:1775766787.127758:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766787.127762:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766787.127767:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766787.127778:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766787.127783:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098975488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766787.127795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766787.127798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766787.127801:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766787.127805:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4248 00000100:00080000:3.0:1775766787.127810:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766787.127814:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766787.127820:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766787.127836:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766787/real 1775766787] req@ffff8b23864f0700 x1862022098975104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766787.127843:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098975616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766787.127868:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098975104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766787.127880:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766787.127886:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766787.127891:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098975104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766787.127923:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098975232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766787.127955:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766787.127969:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766787/real 1775766787] req@ffff8b23864f1180 x1862022098975232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766787.127988:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098975232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766787.127996:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766787.128000:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766787.128003:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098975232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766787.128011:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098975360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766787.128033:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766787.128044:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766787/real 1775766787] req@ffff8b23864f3b80 x1862022098975360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766787.128058:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098975360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766787.128067:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766787.128072:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766787.128075:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098975360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766787.128081:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098975488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766787.128108:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766787.128120:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766787/real 1775766787] req@ffff8b23864f1c00 x1862022098975488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766787.128134:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098975488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766787.128151:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766787.128155:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766787.128158:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098975488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766787.128164:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098975616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766787.128186:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766787.128197:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766787/real 1775766787] req@ffff8b23864f1f80 x1862022098975616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766787.128210:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098975616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766787.128220:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766787.128225:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766787.128228:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098975616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766788.023766:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766789.047769:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766790.071825:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766791.095636:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766792.119752:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766792.247534:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766792.247542:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766792.247546:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766792.247555:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4253 00000100:00080000:3.0:1775766792.247565:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766792.247571:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766792.247620:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098975744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766792.247647:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766792.247650:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766792.247652:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766792.247655:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4253 00000100:00080000:3.0:1775766792.247657:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766792.247659:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766792.247672:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766792.247697:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098975872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766792.247711:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766792.247716:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766792.247718:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766792.247735:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4253 00000100:00080000:3.0:1775766792.247741:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766792.247746:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766792.247753:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766792.247773:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098976000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766792.247776:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098975744:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766792.247804:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766792.247807:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766792.247808:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766792.247813:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4253 00000100:00080000:3.0:1775766792.247818:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766792.247822:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766792.247828:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766792.247839:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098976128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766792.247847:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766792.247850:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766792.247851:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766792.247855:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4253 00000100:00080000:3.0:1775766792.247857:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766792.247859:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766792.247863:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766792.247869:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098976256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766792.247892:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766792.247951:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766792/real 1775766792] req@ffff8b23864f0380 x1862022098975744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766792.247973:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098975744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766792.247992:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766792.247997:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766792.248001:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098975744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766792.248009:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098975872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766792.248031:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766792.248043:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766792/real 1775766792] req@ffff8b23864f2680 x1862022098975872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766792.248055:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098975872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766792.248062:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766792.248065:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766792.248068:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098975872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766792.248098:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098976000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766792.248113:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766792.248122:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766792/real 1775766792] req@ffff8b23864f0a80 x1862022098976000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766792.248133:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098976000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766792.248141:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766792.248144:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766792.248147:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098976000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766792.248153:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098976128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766792.248169:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766792.248177:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766792/real 1775766792] req@ffff8b23864f1f80 x1862022098976128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766792.248188:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098976128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766792.248195:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766792.248197:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766792.248200:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098976128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766792.248206:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098976256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766792.248220:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766792.248227:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766792/real 1775766792] req@ffff8b23864f1c00 x1862022098976256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766792.248237:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098976256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766792.248244:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766792.248247:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766792.248249:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098976256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766793.143831:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766794.167681:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766795.191760:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766796.215610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766797.239836:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766797.367728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766797.367740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766797.367747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766797.367759:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4258 00000100:00080000:1.0:1775766797.367774:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766797.367784:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766797.367839:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098976384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766797.367889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766797.367897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766797.367900:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766797.367909:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4258 00000100:00080000:1.0:1775766797.367916:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766797.367924:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766797.367941:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766797.367972:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098976512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766797.367996:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766797.368000:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766797.368003:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766797.368012:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4258 00000100:00100000:0.0:1775766797.368025:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098976384:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766797.368025:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766797.368048:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766797.368071:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766797.368096:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098976640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766797.368111:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766797.368117:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766797.368120:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766797.368127:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4258 00000100:00080000:1.0:1775766797.368131:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766797.368136:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766797.368142:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766797.368159:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098976768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766797.368170:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766797.368175:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766797.368178:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766797.368183:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4258 00000100:00080000:1.0:1775766797.368187:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766797.368191:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766797.368197:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766797.368207:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766797.368223:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098976896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766797.368309:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766797/real 1775766797] req@ffff8b23864fce00 x1862022098976384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766797.368338:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098976384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766797.368355:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766797.368361:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766797.368367:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098976384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766797.368405:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098976512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766797.368436:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766797.368525:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766797/real 1775766797] req@ffff8b23864fd180 x1862022098976512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766797.368544:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098976512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766797.368555:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766797.368559:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766797.368564:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098976512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766797.368573:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098976640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766797.368600:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766797.368614:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766797/real 1775766797] req@ffff8b23864fea00 x1862022098976640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766797.368668:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098976640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766797.368679:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766797.368683:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766797.368688:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098976640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766797.368696:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098976768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766797.368721:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766797.368734:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766797/real 1775766797] req@ffff8b23864fca80 x1862022098976768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766797.368748:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098976768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766797.368757:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766797.368761:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766797.368764:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098976768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766797.368771:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098976896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766797.368792:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766797.368802:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766797/real 1775766797] req@ffff8b23864fe680 x1862022098976896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766797.368816:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098976896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766797.368835:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766797.368839:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766797.368843:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098976896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766798.263720:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766799.287697:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766800.311687:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766801.335718:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766802.359663:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766802.487625:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766802.487635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766802.487639:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766802.487651:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4263 00000100:00080000:3.0:1775766802.487683:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766802.487693:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766802.487748:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098977024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766802.487790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766802.487799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766802.487802:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766802.487807:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4263 00000100:00080000:3.0:1775766802.487811:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766802.487816:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766802.487833:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766802.487869:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098977152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766802.487886:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766802.487894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775766802.487895:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098977024:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766802.487897:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766802.487914:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4263 00000100:00080000:3.0:1775766802.487919:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766802.487925:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766802.487931:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766802.487948:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098977280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766802.487957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766802.487961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766802.487962:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766802.487977:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4263 00000100:00080000:3.0:1775766802.487980:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766802.487984:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766802.487989:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766802.487995:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766802.487999:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098977408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766802.488010:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766802.488013:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766802.488016:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766802.488019:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4263 00000100:00080000:3.0:1775766802.488021:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766802.488024:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766802.488029:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766802.488045:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098977536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766802.488066:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766802/real 1775766802] req@ffff8b23864f3b80 x1862022098977024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766802.488086:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098977024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766802.488095:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766802.488098:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766802.488101:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098977024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766802.488122:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098977152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766802.488139:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766802.488147:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766802/real 1775766802] req@ffff8b23864f1180 x1862022098977152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766802.488155:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098977152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766802.488164:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766802.488166:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766802.488169:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098977152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766802.488173:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098977280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766802.488183:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766802.488188:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766802/real 1775766802] req@ffff8b23864f0700 x1862022098977280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766802.488195:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098977280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766802.488200:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766802.488202:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766802.488204:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098977280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766802.488208:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098977408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766802.488218:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766802.488224:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766802/real 1775766802] req@ffff8b23864f3100 x1862022098977408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766802.488231:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098977408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766802.488235:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766802.488237:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766802.488240:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098977408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766802.488244:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098977536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766802.488253:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766802.488258:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766802/real 1775766802] req@ffff8b23864f2d80 x1862022098977536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766802.488265:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098977536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766802.488270:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766802.488272:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766802.488274:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098977536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766803.383735:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766804.407685:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766805.431686:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766806.455696:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766807.479753:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766807.607616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766807.607625:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766807.607630:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766807.607640:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4269 00000100:00080000:0.0:1775766807.607651:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766807.607657:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766807.607707:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098977664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766807.607737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766807.607740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766807.607742:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766807.607746:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4269 00000100:00080000:0.0:1775766807.607749:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766807.607752:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766807.607762:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766807.607771:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098977792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766807.607782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766807.607785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766807.607786:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766807.607790:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4269 00000100:00080000:0.0:1775766807.607792:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766807.607795:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766807.607799:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766807.607808:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098977664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766807.607809:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098977920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766807.607822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766807.607825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766807.607839:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766807.607842:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4269 00000100:00080000:0.0:1775766807.607844:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766807.607846:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766807.607850:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766807.607861:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098978048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766807.607869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766807.607871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766807.607873:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766807.607875:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4269 00000100:00080000:0.0:1775766807.607877:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766807.607880:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766807.607884:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766807.607893:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098978176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766807.607909:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766807.607951:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766807/real 1775766807] req@ffff8b2382daed80 x1862022098977664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766807.607968:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098977664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766807.607978:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766807.607981:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766807.607985:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098977664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766807.607991:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098977792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766807.608009:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766807.608018:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766807/real 1775766807] req@ffff8b2382daea00 x1862022098977792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766807.608027:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098977792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766807.608033:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766807.608036:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766807.608038:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098977792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766807.608059:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098977920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766807.608072:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766807.608079:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766807/real 1775766807] req@ffff8b2382dad500 x1862022098977920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766807.608088:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098977920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766807.608094:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766807.608097:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766807.608099:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098977920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766807.608103:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098978048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766807.608115:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766807.608122:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766807/real 1775766807] req@ffff8b2382dae680 x1862022098978048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766807.608136:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098978048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766807.608142:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766807.608144:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766807.608146:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098978048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766807.608150:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098978176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766807.608162:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766807.608168:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766807/real 1775766807] req@ffff8b2382dac380 x1862022098978176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766807.608175:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098978176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766807.608182:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766807.608184:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766807.608186:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098978176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766808.503601:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766809.527595:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766810.551773:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766811.575749:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766812.599689:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766812.727585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766812.727594:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766812.727598:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766812.727609:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4274 00000100:00080000:2.0:1775766812.727626:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766812.727632:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766812.727670:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022098978304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766812.727695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766812.727698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766812.727700:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766812.727702:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4274 00000100:00080000:2.0:1775766812.727705:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766812.727708:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766812.727715:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766812.727724:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022098978432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766812.727734:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766812.727737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766812.727739:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766812.727743:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4274 00000100:00080000:2.0:1775766812.727745:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766812.727748:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766812.727751:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766812.727761:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022098978560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766812.727779:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098978304:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775766812.727784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766812.727793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766812.727795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766812.727801:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4274 00000100:00080000:2.0:1775766812.727807:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766812.727814:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766812.727821:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766812.727844:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022098978688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766812.727852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766812.727855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766812.727857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766812.727860:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4274 00000100:00080000:2.0:1775766812.727866:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766812.727869:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766812.727873:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766812.727880:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775766812.727885:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022098978816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766812.727944:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766812/real 1775766812] req@ffff8b23b9ea5500 x1862022098978304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766812.727964:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022098978304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766812.727981:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766812.727985:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766812.727989:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098978304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766812.727996:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098978432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766812.728028:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766812.728038:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766812/real 1775766812] req@ffff8b23b9ea4700 x1862022098978432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766812.728048:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022098978432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766812.728054:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766812.728057:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766812.728059:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098978432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766812.728085:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098978560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766812.728098:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766812.728116:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766812/real 1775766812] req@ffff8b23b9ea6a00 x1862022098978560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766812.728124:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022098978560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766812.728131:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766812.728133:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766812.728136:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098978560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766812.728141:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098978688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766812.728153:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766812.728159:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766812/real 1775766812] req@ffff8b23b9ea4000 x1862022098978688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766812.728167:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022098978688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766812.728173:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766812.728175:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766812.728177:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098978688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766812.728181:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098978816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766812.728193:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766812.728198:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766812/real 1775766812] req@ffff8b23b9ea5880 x1862022098978816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766812.728205:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022098978816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766812.728211:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766812.728214:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766812.728216:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098978816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766813.623671:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766814.647749:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766815.671661:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766816.695716:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766817.719677:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766817.847644:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766817.847659:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766817.847666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766817.847680:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4279 00000100:00080000:0.0:1775766817.847697:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766817.847706:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766817.847782:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098978944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766817.847827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766817.847834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766817.847838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766817.847845:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4279 00000100:00080000:0.0:1775766817.847852:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766817.847858:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766817.847874:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766817.847905:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098979072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766817.847907:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098978944:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766817.847947:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766817.847952:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766817.847954:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766817.847962:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4279 00000100:00080000:0.0:1775766817.847968:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766817.847974:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766817.847983:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766817.848003:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098979200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766817.848015:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766817.848019:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766817.848027:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775766817.848028:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766817.848038:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4279 00000100:00080000:0.0:1775766817.848045:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766817.848052:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766817.848060:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766817.848077:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098979328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766817.848091:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766817.848111:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766817.848116:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766817.848116:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766817/real 1775766817] req@ffff8b2382dad880 x1862022098978944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766817.848136:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4279 00000100:00080000:0.0:1775766817.848155:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766817.848162:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766817.848170:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766817.848170:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098978944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766817.848193:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1775766817.848200:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098979456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766817.848204:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766817.848221:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098978944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766817.848285:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098979072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766817.848309:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766817.848319:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766817/real 1775766817] req@ffff8b2382daca80 x1862022098979072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766817.848328:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098979072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766817.848334:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766817.848336:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766817.848338:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098979072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766817.848343:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098979200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766817.848354:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766817.848359:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766817/real 1775766817] req@ffff8b2382daf100 x1862022098979200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766817.848366:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098979200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766817.848377:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766817.848379:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766817.848381:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098979200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766817.848385:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098979328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766817.848397:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766817.848402:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766817/real 1775766817] req@ffff8b2382dac000 x1862022098979328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766817.848409:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098979328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766817.848431:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766817.848434:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766817.848436:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098979328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766817.848441:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098979456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766817.848479:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766817.848488:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766817/real 1775766817] req@ffff8b2382dac380 x1862022098979456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766817.848496:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098979456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766817.848501:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766817.848504:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766817.848507:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098979456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766818.743655:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766819.767607:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766820.791603:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766821.815592:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766822.839603:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766822.967570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766822.967582:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766822.967588:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766822.967599:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4284 00000100:00080000:2.0:1775766822.967614:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766822.967622:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766822.967667:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022098979584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766822.967696:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766822.967699:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766822.967701:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766822.967705:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4284 00000100:00080000:2.0:1775766822.967708:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766822.967711:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766822.967719:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766822.967729:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098979712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766822.967739:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766822.967742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766822.967744:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766822.967748:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4284 00000100:00080000:2.0:1775766822.967751:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766822.967754:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766822.967759:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766822.967765:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098979584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775766822.967772:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098979840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766822.967781:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766822.967784:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766822.967785:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766822.967789:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4284 00000100:00080000:2.0:1775766822.967791:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766822.967794:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766822.967797:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766822.967808:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098979968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766822.967815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766822.967818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766822.967819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766822.967822:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4284 00000100:00080000:2.0:1775766822.967832:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766822.967836:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766822.967839:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766822.967848:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775766822.967852:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098980096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766822.967900:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766822/real 1775766822] req@ffff8b23b9ea4a80 x1862022098979584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766822.967921:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022098979584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766822.967931:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766822.967934:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766822.967938:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098979584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766822.967944:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098979712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766822.967963:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766822.967972:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766822/real 1775766822] req@ffff8b23b9ea7100 x1862022098979712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766822.967982:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022098979712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766822.967988:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766822.967990:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766822.967993:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098979712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766822.968012:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098979840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766822.968025:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766822.968033:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766822/real 1775766822] req@ffff8b23b9ea4380 x1862022098979840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766822.968042:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022098979840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766822.968048:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766822.968050:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766822.968052:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098979840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766822.968057:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098979968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766822.968069:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766822.968076:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766822/real 1775766822] req@ffff8b23b9ea4e00 x1862022098979968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766822.968084:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022098979968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766822.968090:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766822.968092:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766822.968094:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098979968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766822.968098:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098980096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766822.968109:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766822.968119:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766822/real 1775766822] req@ffff8b23b9ea7480 x1862022098980096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766822.968127:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022098980096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766822.968134:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766822.968138:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766822.968142:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098980096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766823.863604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766824.887566:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766825.911614:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766826.935666:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766827.959602:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766828.087560:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766828.087568:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766828.087572:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766828.087581:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4289 00000100:00080000:1.0:1775766828.087590:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766828.087595:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766828.087630:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098980224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766828.087654:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766828.087657:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766828.087658:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766828.087661:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4289 00000100:00080000:1.0:1775766828.087663:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766828.087665:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766828.087672:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766828.087681:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098980352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766828.087689:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766828.087692:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766828.087694:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766828.087698:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4289 00000100:00080000:1.0:1775766828.087701:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766828.087704:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766828.087707:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766828.087720:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098980480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766828.087720:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098980224:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766828.087734:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766828.087737:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766828.087738:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766828.087740:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4289 00000100:00080000:1.0:1775766828.087742:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766828.087744:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766828.087748:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766828.087764:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098980608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766828.087771:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766828.087773:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766828.087774:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766828.087776:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4289 00000100:00080000:1.0:1775766828.087784:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766828.087787:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766828.087791:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766828.087795:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766828.087804:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098980736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766828.087842:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766828/real 1775766828] req@ffff8b23864fdc00 x1862022098980224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766828.087857:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098980224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766828.087865:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766828.087869:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766828.087872:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098980224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766828.087878:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098980352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766828.087894:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766828.087902:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766828/real 1775766828] req@ffff8b23864fd500 x1862022098980352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766828.087909:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098980352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766828.087915:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766828.087917:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766828.087919:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098980352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766828.087938:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098980480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766828.087949:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766828.087954:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766828/real 1775766828] req@ffff8b23864ff100 x1862022098980480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766828.087962:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098980480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766828.087967:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766828.087969:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766828.087971:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098980480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766828.087974:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098980608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766828.087984:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766828.087989:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766828/real 1775766828] req@ffff8b23864fe300 x1862022098980608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766828.087996:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098980608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766828.088001:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766828.088003:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766828.088004:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098980608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766828.088008:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098980736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766828.088016:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766828.088021:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766828/real 1775766828] req@ffff8b23864ffb80 x1862022098980736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766828.088029:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098980736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766828.088034:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766828.088035:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766828.088037:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098980736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766828.983595:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766830.007600:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766831.031675:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766832.055806:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766833.079650:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766833.207629:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766833.207642:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766833.207647:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766833.207659:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4294 00000100:00080000:3.0:1775766833.207674:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766833.207682:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766833.207744:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098980864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766833.207777:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766833.207781:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766833.207782:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766833.207786:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4294 00000100:00080000:3.0:1775766833.207789:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766833.207792:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766833.207801:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766833.207811:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098980992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766833.207832:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766833.207835:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766833.207837:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766833.207842:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4294 00000100:00080000:3.0:1775766833.207845:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766833.207848:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766833.207851:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766833.207859:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098981120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766833.207868:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766833.207871:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766833.207872:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766833.207877:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4294 00000100:00080000:3.0:1775766833.207879:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766833.207885:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766833.207888:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098980864:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775766833.207889:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766833.207908:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098981248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766833.207916:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766833.207918:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766833.207920:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766833.207923:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4294 00000100:00080000:3.0:1775766833.207927:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766833.207930:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766833.207934:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766833.207942:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098981376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766833.207974:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766833.208012:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766833/real 1775766833] req@ffff8b23864f0000 x1862022098980864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766833.208027:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098980864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766833.208040:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766833.208044:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766833.208047:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098980864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766833.208052:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098980992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766833.208066:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766833.208074:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766833/real 1775766833] req@ffff8b23864f2a00 x1862022098980992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766833.208082:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098980992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766833.208086:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766833.208089:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766833.208091:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098980992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766833.208110:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098981120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766833.208121:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766833.208126:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766833/real 1775766833] req@ffff8b23864f3480 x1862022098981120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766833.208133:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098981120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766833.208139:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766833.208141:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766833.208160:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098981120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766833.208165:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098981248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766833.208179:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766833.208186:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766833/real 1775766833] req@ffff8b23864f2d80 x1862022098981248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766833.208194:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098981248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766833.208200:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766833.208202:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766833.208204:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098981248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766833.208224:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098981376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766833.208236:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766833.208241:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766833/real 1775766833] req@ffff8b23864f3100 x1862022098981376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766833.208247:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098981376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766833.208280:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766833.208282:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766833.208284:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098981376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766834.103659:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766835.127691:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766836.151652:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766837.175652:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766838.199708:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766838.327597:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766838.327611:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766838.327618:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766838.327631:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4299 00000100:00080000:3.0:1775766838.327645:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766838.327655:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766838.327722:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098981504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766838.327769:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766838.327774:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766838.327776:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766838.327782:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4299 00000100:00080000:3.0:1775766838.327788:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766838.327793:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766838.327806:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766838.327826:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098981632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766838.327842:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766838.327846:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098981504:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766838.327862:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766838.327869:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766838.327875:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4299 00000100:00080000:3.0:1775766838.327879:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766838.327883:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766838.327889:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766838.327904:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098981760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766838.327916:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766838.327920:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766838.327922:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766838.327927:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4299 00000100:00080000:3.0:1775766838.327931:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766838.327935:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766838.327939:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766838.327953:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098981888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766838.327964:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766838.327975:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766838.327993:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766838.327996:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766838.328002:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4299 00000100:00080000:3.0:1775766838.328008:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766838.328013:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766838.328037:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766838.328041:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766838/real 1775766838] req@ffff8b23864f0700 x1862022098981504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766838.328054:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098982016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766838.328071:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098981504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766838.328085:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766838.328091:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766838.328107:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098981504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766838.328167:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098981632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766838.328246:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766838.328263:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766838/real 1775766838] req@ffff8b23864f1180 x1862022098981632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766838.328276:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098981632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766838.328285:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766838.328288:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766838.328292:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098981632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766838.328299:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098981760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766838.328319:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766838.328329:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766838/real 1775766838] req@ffff8b23864f3b80 x1862022098981760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766838.328347:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098981760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766838.328355:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766838.328358:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766838.328361:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098981760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766838.328377:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098981888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766838.328395:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766838.328405:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766838/real 1775766838] req@ffff8b23864f1c00 x1862022098981888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766838.328417:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098981888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766838.328429:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766838.328435:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766838.328440:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098981888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766838.328547:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098982016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766838.328584:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766838.328600:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766838/real 1775766838] req@ffff8b23864f1f80 x1862022098982016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766838.328614:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098982016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766838.328632:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766838.328635:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766838.328639:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098982016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766839.223644:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766840.247733:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766841.271596:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766842.295629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766843.319683:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766843.447580:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766843.447591:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766843.447596:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766843.447607:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4304 00000100:00080000:0.0:1775766843.447621:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766843.447628:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766843.447673:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098982144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766843.447716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766843.447719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766843.447721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766843.447724:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4304 00000100:00080000:0.0:1775766843.447726:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766843.447729:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766843.447739:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766843.447748:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098982272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766843.447754:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098982144:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766843.447770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766843.447773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766843.447774:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766843.447779:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4304 00000100:00080000:0.0:1775766843.447781:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766843.447784:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766843.447787:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766843.447796:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098982400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766843.447803:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766843.447806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766843.447807:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766843.447809:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4304 00000100:00080000:0.0:1775766843.447812:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766843.447814:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766843.447817:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766843.447826:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098982528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766843.447831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1775766843.447834:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766843.447836:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766843.447838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766843.447842:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4304 00000100:00080000:0.0:1775766843.447846:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766843.447849:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766843.447852:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766843.447865:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098982656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766843.447891:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766843/real 1775766843] req@ffff8b2382dae680 x1862022098982144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766843.447914:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098982144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766843.447926:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766843.447929:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766843.447933:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098982144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766843.447959:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098982272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766843.447979:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766843.447989:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766843/real 1775766843] req@ffff8b2382dad500 x1862022098982272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766843.447997:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098982272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766843.448003:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766843.448005:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766843.448007:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098982272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766843.448011:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098982400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766843.448023:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766843.448029:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766843/real 1775766843] req@ffff8b2382daea00 x1862022098982400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766843.448037:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098982400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766843.448042:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766843.448044:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766843.448046:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098982400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766843.448049:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098982528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766843.448060:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766843.448066:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766843/real 1775766843] req@ffff8b2382daed80 x1862022098982528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766843.448073:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098982528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766843.448078:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766843.448079:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766843.448081:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098982528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766843.448084:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098982656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766843.448095:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766843.448099:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766843/real 1775766843] req@ffff8b2382dace00 x1862022098982656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766843.448111:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098982656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766843.448116:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766843.448118:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766843.448120:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098982656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766844.343658:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766845.367572:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766846.391652:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766847.415598:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766848.439717:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766848.567712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766848.567727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766848.567735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766848.567751:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4309 00000100:00080000:0.0:1775766848.567771:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766848.567784:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766848.567847:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098982784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766848.567894:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766848.567901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766848.567904:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766848.567913:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4309 00000100:00080000:0.0:1775766848.567922:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766848.567931:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766848.567948:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766848.567949:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098982784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766848.567985:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098982912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766848.568038:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766848.568043:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766848.568045:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766848.568051:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4309 00000100:00080000:0.0:1775766848.568059:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766848.568064:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766848.568072:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766848.568153:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098983040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766848.568168:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766848.568173:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766848.568176:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766848.568181:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4309 00000100:00080000:0.0:1775766848.568195:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766848.568201:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766848.568208:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098982912:192.168.202.128@tcp:38:kworker.0 00000008:00100000:0.0:1775766848.568212:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766848.568236:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098983168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766848.568250:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766848.568254:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766848.568257:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766848.568263:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4309 00000100:00080000:0.0:1775766848.568270:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775766848.568280:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098983040:192.168.202.128@tcp:38:kworker.0 00000100:00080000:0.0:1775766848.568281:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766848.568311:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766848.568344:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098983296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766848.568368:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098983168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766848.568413:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098983296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766849.463739:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766850.487556:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766851.511723:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775766851.639862:0:2365:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775766851.639914:0:2365:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775766851.639935:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766851.639983:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766851.639995:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766851.640004:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766851.640015:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:3.0:1775766851.640074:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766848/real 1775766851] req@ffff8b2382dace00 x1862022098983296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766851.640124:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098983296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766851.640141:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766851.640149:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766851.640155:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098983296:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766851.640165:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766848/real 1775766851] req@ffff8b2382dadf80 x1862022098983168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766851.640191:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098983168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766851.640202:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766851.640206:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766851.640209:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098983168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766851.640219:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766848/real 1775766851] req@ffff8b2382dafb80 x1862022098982912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766851.640233:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098982912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766851.640243:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766851.640247:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766851.640251:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098982912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766851.640260:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766848/real 1775766851] req@ffff8b2382dac700 x1862022098983040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766851.640273:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098983040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766851.640282:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766851.640286:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766851.640290:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098983040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766851.640298:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766848/real 1775766851] req@ffff8b2382daf480 x1862022098982784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766851.640314:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098982784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766851.640322:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766851.640327:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766851.640331:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098982784:192.168.202.128@tcp:250:kworker.0 00010000:00010000:1.0:1775766852.535635:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766853.559673:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766853.687611:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766853.687624:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766853.687630:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766853.687645:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4315 00000100:00080000:2.0:1775766853.687661:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766853.687670:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775766853.687717:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022098983424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766853.687752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766853.687757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766853.687759:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766853.687765:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4315 00000100:00080000:2.0:1775766853.687770:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766853.687775:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766853.687785:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098983424:192.168.202.128@tcp:250:kworker.0 00000008:00100000:2.0:1775766853.687787:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766853.687808:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022098983552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766853.687822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766853.687826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766853.687828:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766853.687834:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4315 00000100:00080000:2.0:1775766853.687838:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766853.687844:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766853.687850:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766853.687865:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022098983680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766853.687877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766853.687881:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766853.687883:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766853.687888:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4315 00000800:00000100:3.0:1775766853.687894:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775766853.687895:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766853.687903:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775766853.687910:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766853.687922:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022098983808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766853.687940:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766853.687944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775766853.687946:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766853.687951:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4315 00000100:00080000:2.0:1775766853.687955:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766853.687962:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766853.687963:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766853/real 1775766853] req@ffff8b23b9ea6300 x1862022098983424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1775766853.687969:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766853.687986:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022098983936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766853.687997:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022098983424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766853.688011:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766853.688018:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766853.688023:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098983424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766853.688093:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098983552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766853.688123:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766853.688137:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766853/real 1775766853] req@ffff8b23b9ea7480 x1862022098983552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766853.688150:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022098983552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766853.688159:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766853.688164:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766853.688168:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098983552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766853.688174:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098983680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766853.688197:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766853.688209:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766853/real 1775766853] req@ffff8b23b9ea4e00 x1862022098983680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766853.688222:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022098983680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766853.688232:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766853.688236:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766853.688240:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098983680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766853.688247:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098983808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766853.688269:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766853.688279:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766853/real 1775766853] req@ffff8b23b9ea4380 x1862022098983808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766853.688292:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022098983808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766853.688301:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766853.688305:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766853.688308:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098983808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766853.688315:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098983936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766853.688336:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766853.688347:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766853/real 1775766853] req@ffff8b23b9ea7100 x1862022098983936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766853.688360:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022098983936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766853.688368:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766853.688371:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766853.688374:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098983936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766854.583668:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766855.607726:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766856.631616:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766857.655641:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766858.679698:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766858.807555:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766858.807563:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766858.807567:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766858.807575:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4320 00000100:00080000:3.0:1775766858.807585:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766858.807590:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766858.807625:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098984064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766858.807650:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766858.807652:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766858.807654:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766858.807656:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4320 00000100:00080000:3.0:1775766858.807659:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766858.807661:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766858.807669:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766858.807677:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098984192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766858.807684:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766858.807687:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766858.807688:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766858.807690:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4320 00000100:00080000:3.0:1775766858.807692:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766858.807694:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766858.807697:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766858.807703:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098984320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766858.807708:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766858.807711:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766858.807712:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766858.807715:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4320 00000100:00080000:3.0:1775766858.807717:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766858.807720:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766858.807726:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766858.807733:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098984448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766858.807742:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766858.807745:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766858.807747:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766858.807752:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4320 00000100:00080000:3.0:1775766858.807755:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766858.807759:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766858.807764:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766858.807777:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098984064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766858.807779:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098984576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766858.807865:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766858.807912:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766858/real 1775766858] req@ffff8b23864f0a80 x1862022098984064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766858.807928:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098984064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766858.807937:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766858.807941:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766858.807944:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098984064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766858.807950:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098984192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766858.807966:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766858.808005:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766858/real 1775766858] req@ffff8b23864f2680 x1862022098984192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766858.808021:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098984192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766858.808027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766858.808029:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766858.808031:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098984192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766858.808036:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098984320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766858.808053:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766858.808061:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766858/real 1775766858] req@ffff8b23864f0380 x1862022098984320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766858.808068:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098984320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766858.808073:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766858.808075:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766858.808077:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098984320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766858.808081:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098984448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766858.808093:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766858.808098:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766858/real 1775766858] req@ffff8b23864f1f80 x1862022098984448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766858.808106:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098984448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766858.808116:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766858.808118:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766858.808120:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098984448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766858.808145:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098984576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766858.808155:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766858.808160:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766858/real 1775766858] req@ffff8b23864f1c00 x1862022098984576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766858.808167:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098984576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766858.808172:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766858.808174:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766858.808176:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098984576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766859.703769:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766860.727672:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766861.751651:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766862.775626:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766863.799799:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766863.927605:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766863.927614:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766863.927618:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766863.927627:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4325 00000100:00080000:0.0:1775766863.927636:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766863.927660:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766863.927724:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098984704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766863.927786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766863.927791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766863.927793:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766863.927797:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4325 00000100:00080000:0.0:1775766863.927800:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766863.927804:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766863.927815:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766863.927829:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098984832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766863.927844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766863.927848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766863.927865:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766863.927876:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4325 00000100:00100000:3.0:1775766863.927876:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098984704:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766863.927890:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766863.927898:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766863.927905:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766863.927927:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098984960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766863.927950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766863.927953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766863.927955:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766863.927959:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4325 00000100:00080000:0.0:1775766863.927964:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766863.927968:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766863.927973:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766863.927983:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098985088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766863.927989:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766863.927991:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766863.927992:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766863.927995:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4325 00000100:00080000:0.0:1775766863.927997:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766863.927999:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766863.928003:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766863.928015:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098985216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766863.928036:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766863.928113:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766863/real 1775766863] req@ffff8b2382daed80 x1862022098984704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766863.928141:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098984704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766863.928157:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766863.928164:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766863.928170:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098984704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766863.928207:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098984832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766863.928241:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766863.928264:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766863/real 1775766863] req@ffff8b2382daea00 x1862022098984832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766863.928288:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098984832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766863.928304:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766863.928312:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766863.928319:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098984832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766863.928333:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098984960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766863.928377:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766863.928400:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766863/real 1775766863] req@ffff8b2382dad500 x1862022098984960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766863.928416:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098984960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766863.928427:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766863.928431:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766863.928435:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098984960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766863.928443:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098985088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766863.928513:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766863.928527:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766863/real 1775766863] req@ffff8b2382dae680 x1862022098985088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766863.928542:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098985088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766863.928552:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766863.928557:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766863.928561:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098985088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766863.928568:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098985216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766863.928590:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766863.928600:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766863/real 1775766863] req@ffff8b2382dac380 x1862022098985216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766863.928614:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098985216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766863.928624:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766863.928628:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766863.928631:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098985216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766864.823657:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766865.847734:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766866.871619:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766867.895681:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766868.919592:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766869.047573:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766869.047588:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766869.047593:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766869.047606:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4330 00000100:00080000:0.0:1775766869.047622:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766869.047630:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766869.047679:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098985344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766869.047714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766869.047717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766869.047718:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766869.047721:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4330 00000100:00080000:0.0:1775766869.047724:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766869.047727:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766869.047737:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766869.047746:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098985472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766869.047755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766869.047759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766869.047760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766869.047767:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4330 00000100:00080000:0.0:1775766869.047770:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766869.047773:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766869.047776:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766869.047786:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098985600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766869.047796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766869.047798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766869.047800:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766869.047807:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4330 00000100:00100000:3.0:1775766869.047807:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098985344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766869.047816:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766869.047826:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766869.047830:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766869.047840:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098985728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766869.047846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766869.047849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766869.047850:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766869.047853:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4330 00000100:00080000:0.0:1775766869.047855:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766869.047858:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766869.047862:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766869.047869:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098985856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766869.047906:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766869.047956:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766869/real 1775766869] req@ffff8b2382dac000 x1862022098985344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766869.047975:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098985344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766869.047984:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766869.047988:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766869.047990:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098985344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766869.047996:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098985472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766869.048013:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766869.048021:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766869/real 1775766869] req@ffff8b2382daf100 x1862022098985472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766869.048033:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098985472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766869.048038:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766869.048040:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766869.048042:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098985472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766869.048063:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098985600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766869.048083:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766869.048090:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766869/real 1775766869] req@ffff8b2382daca80 x1862022098985600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766869.048098:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098985600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766869.048103:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766869.048105:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766869.048107:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098985600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766869.048111:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098985728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766869.048122:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766869.048128:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766869/real 1775766869] req@ffff8b2382dad880 x1862022098985728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766869.048135:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098985728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766869.048144:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766869.048146:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766869.048148:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098985728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766869.048152:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098985856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766869.048161:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766869.048166:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766869/real 1775766869] req@ffff8b2382dac380 x1862022098985856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766869.048173:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098985856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766869.048178:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766869.048180:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766869.048181:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098985856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766869.943647:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766870.967618:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766871.991625:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766873.015638:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766874.039647:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766874.167579:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766874.167589:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766874.167594:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766874.167605:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4335 00000100:00080000:0.0:1775766874.167618:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766874.167625:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766874.167674:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098985984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766874.167711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766874.167715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766874.167717:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766874.167722:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4335 00000100:00080000:0.0:1775766874.167726:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766874.167731:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766874.167743:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766874.167757:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098986112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766874.167773:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766874.167773:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098985984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766874.167791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766874.167793:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766874.167800:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4335 00000100:00080000:0.0:1775766874.167804:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766874.167810:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766874.167816:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766874.167853:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098986240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766874.167865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766874.167870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766874.167872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775766874.167879:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766874.167885:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4335 00000100:00080000:0.0:1775766874.167891:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766874.167896:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766874.167903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766874.167917:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098986368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766874.167929:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766874.167935:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766874/real 1775766874] req@ffff8b2382dae680 x1862022098985984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766874.167937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766874.167943:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766874.167960:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4335 00000100:00080000:0.0:1775766874.167970:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766874.167976:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766874.167981:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098985984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775766874.167984:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775766874.168004:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1775766874.168005:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098986496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766874.168021:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766874.168036:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098985984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766874.168066:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098986112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766874.168087:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766874.168097:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766874/real 1775766874] req@ffff8b2382dad500 x1862022098986112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766874.168107:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098986112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766874.168113:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766874.168116:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766874.168118:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098986112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766874.168123:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098986240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766874.168134:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766874.168140:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766874/real 1775766874] req@ffff8b2382daea00 x1862022098986240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766874.168148:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098986240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766874.168153:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766874.168155:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766874.168157:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098986240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766874.168161:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098986368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766874.168173:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766874.168178:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766874/real 1775766874] req@ffff8b2382daed80 x1862022098986368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766874.168186:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098986368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766874.168191:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766874.168193:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766874.168195:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098986368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766874.168198:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098986496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766874.168208:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766874.168213:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766874/real 1775766874] req@ffff8b2382daf480 x1862022098986496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766874.168220:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098986496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766874.168225:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766874.168227:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766874.168229:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098986496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766875.063633:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766876.087670:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766877.111629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766878.135612:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766879.159684:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766879.287613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766879.287624:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766879.287629:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766879.287641:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4340 00000100:00080000:3.0:1775766879.287653:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766879.287661:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766879.287706:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098986624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766879.287745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766879.287749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766879.287751:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766879.287785:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4340 00000100:00080000:3.0:1775766879.287790:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766879.287795:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766879.287807:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766879.287824:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098986624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766879.287826:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098986752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766879.287850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766879.287854:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766879.287856:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766879.287863:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4340 00000100:00080000:3.0:1775766879.287869:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766879.287874:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766879.287880:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766879.287905:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098986880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766879.287915:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766879.287929:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766879.287934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766879.287936:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766879.287944:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4340 00000100:00080000:3.0:1775766879.287951:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766879.287956:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766879.287962:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766879.287973:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766879/real 1775766879] req@ffff8b23864f3b80 x1862022098986624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766879.287991:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098987008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766879.287993:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098986624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766879.288029:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766879.288037:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00080000:3.0:1775766879.288037:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766879.288046:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098986624:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766879.288051:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766879.288055:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766879.288062:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4340 00000100:00080000:3.0:1775766879.288067:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766879.288072:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766879.288076:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098986752:192.168.202.128@tcp:38:kworker.0 00000008:00100000:3.0:1775766879.288086:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766879.288098:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766879.288109:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766879/real 1775766879] req@ffff8b23864f1180 x1862022098986752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766879.288120:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098987136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766879.288124:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098986752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766879.288162:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766879.288171:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766879.288179:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098986752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766879.288190:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098986880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766879.288219:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766879.288238:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766879/real 1775766879] req@ffff8b23864f0700 x1862022098986880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766879.288248:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098986880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766879.288254:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766879.288257:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766879.288259:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098986880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766879.288264:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098987008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766879.288282:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766879.288289:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766879/real 1775766879] req@ffff8b23864f3100 x1862022098987008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766879.288297:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098987008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766879.288302:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766879.288304:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766879.288306:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098987008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766879.288317:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098987136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766879.288328:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766879.288334:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766879/real 1775766879] req@ffff8b23864f2d80 x1862022098987136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766879.288341:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098987136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766879.288346:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766879.288348:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766879.288350:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098987136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766880.183642:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766881.207585:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766882.231624:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766883.255620:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766884.279584:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766884.407537:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766884.407548:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766884.407554:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766884.407565:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4345 00000100:00080000:0.0:1775766884.407576:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766884.407583:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766884.407632:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098987264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766884.407665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766884.407669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766884.407671:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766884.407677:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4345 00000100:00080000:0.0:1775766884.407681:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766884.407686:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766884.407693:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098987264:192.168.202.128@tcp:250:kworker.0 00000008:00100000:0.0:1775766884.407696:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766884.407713:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098987392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766884.407741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766884.407745:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766884.407747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766884.407752:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4345 00000100:00080000:0.0:1775766884.407757:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766884.407762:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766884.407768:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766884.407769:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766884.407786:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098987520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766884.407804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766884.407808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766884.407810:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766884.407813:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766884/real 1775766884] req@ffff8b2382dac700 x1862022098987264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766884.407817:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4345 00000100:00080000:0.0:1775766884.407825:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766884.407831:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766884.407837:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098987264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775766884.407838:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775766884.407851:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766884.407856:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766884.407860:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098987648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766884.407861:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098987264:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766884.407877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766884.407880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766884.407882:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766884.407887:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4345 00000100:00100000:3.0:1775766884.407891:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098987392:192.168.202.128@tcp:38:kworker.0 00000100:00080000:0.0:1775766884.407893:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766884.407902:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766884.407906:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766884.407919:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098987776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766884.407929:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766884.407949:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766884/real 1775766884] req@ffff8b2382dafb80 x1862022098987392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766884.407960:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098987392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766884.407966:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766884.407968:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766884.407971:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098987392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766884.407975:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098987520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766884.407990:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766884.407996:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766884/real 1775766884] req@ffff8b2382dadf80 x1862022098987520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766884.408003:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098987520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766884.408008:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766884.408010:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766884.408012:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098987520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766884.408015:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098987648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766884.408026:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766884.408031:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766884/real 1775766884] req@ffff8b2382dace00 x1862022098987648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766884.408039:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098987648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766884.408043:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766884.408045:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766884.408047:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098987648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766884.408060:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098987776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766884.408069:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766884.408074:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766884/real 1775766884] req@ffff8b2382dac700 x1862022098987776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766884.408080:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098987776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766884.408085:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766884.408087:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766884.408089:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098987776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766885.303658:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766886.327606:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766887.351671:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766888.375718:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766889.399739:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766889.527573:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766889.527603:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766889.527609:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766889.527623:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4350 00000100:00080000:3.0:1775766889.527638:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766889.527645:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766889.527736:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098987904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766889.527764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766889.527768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766889.527769:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766889.527772:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4350 00000100:00080000:3.0:1775766889.527776:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766889.527784:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766889.527795:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766889.527804:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098987904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766889.527806:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098988032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766889.527820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766889.527823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766889.527825:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766889.527828:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4350 00000100:00080000:3.0:1775766889.527831:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766889.527833:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766889.527837:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766889.527845:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098988160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766889.527853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766889.527855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766889.527857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766889.527860:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4350 00000100:00080000:3.0:1775766889.527863:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766889.527866:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766889.527869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766889.527878:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098988288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766889.527883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766889.527887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1775766889.527889:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766889.527890:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766889.527896:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4350 00000100:00080000:3.0:1775766889.527899:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766889.527902:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766889.527905:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766889.527913:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098988416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766889.527936:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766889/real 1775766889] req@ffff8b23864f3480 x1862022098987904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766889.527952:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098987904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766889.527961:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766889.527964:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766889.527968:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098987904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766889.527990:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098988032:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766889.528006:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766889.528014:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766889/real 1775766889] req@ffff8b23864f2a00 x1862022098988032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766889.528022:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098988032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766889.528028:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766889.528030:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766889.528032:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098988032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766889.528036:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098988160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766889.528046:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766889.528052:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766889/real 1775766889] req@ffff8b23864f0000 x1862022098988160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766889.528059:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098988160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766889.528070:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766889.528072:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766889.528074:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098988160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766889.528077:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098988288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766889.528087:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766889.528093:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766889/real 1775766889] req@ffff8b23864f2d80 x1862022098988288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766889.528125:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098988288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766889.528132:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766889.528135:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766889.528137:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098988288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766889.528142:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098988416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766889.528159:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766889.528166:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766889/real 1775766889] req@ffff8b23864f3100 x1862022098988416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766889.528190:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098988416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766889.528195:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766889.528197:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766889.528200:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098988416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766890.423716:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766891.447626:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766892.471585:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766893.495646:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766894.519607:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766894.647573:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766894.647584:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766894.647589:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766894.647600:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4356 00000100:00080000:3.0:1775766894.647637:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766894.647645:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766894.647698:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098988544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766894.647725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766894.647732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766894.647733:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766894.647736:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4356 00000100:00080000:3.0:1775766894.647739:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766894.647741:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766894.647751:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766894.647762:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098988672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766894.647771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766894.647774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766894.647775:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766894.647779:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4356 00000100:00080000:3.0:1775766894.647781:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766894.647785:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766894.647790:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098988544:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775766894.647790:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766894.647800:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098988800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766894.647808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766894.647811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766894.647812:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766894.647815:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4356 00000100:00080000:3.0:1775766894.647817:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766894.647819:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766894.647823:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766894.647831:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098988928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766894.647838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766894.647841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766894.647842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766894.647844:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4356 00000100:00080000:3.0:1775766894.647846:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766894.647848:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766894.647851:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766894.647858:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098989056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766894.647876:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766894.647929:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766894/real 1775766894] req@ffff8b23864f0700 x1862022098988544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766894.647951:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098988544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766894.647964:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766894.647969:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766894.647974:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098988544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766894.647982:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098988672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766894.648008:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766894.648023:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766894/real 1775766894] req@ffff8b23864f1180 x1862022098988672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766894.648036:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098988672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766894.648046:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766894.648049:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766894.648053:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098988672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766894.648088:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098988800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766894.648108:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766894.648119:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766894/real 1775766894] req@ffff8b23864f3b80 x1862022098988800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766894.648132:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098988800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766894.648141:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766894.648145:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766894.648149:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098988800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766894.648155:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098988928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766894.648174:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766894.648184:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766894/real 1775766894] req@ffff8b23864f1c00 x1862022098988928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766894.648196:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098988928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766894.648204:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766894.648208:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766894.648211:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098988928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766894.648218:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098989056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766894.648236:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766894.648245:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766894/real 1775766894] req@ffff8b23864f1f80 x1862022098989056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766894.648258:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098989056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766894.648268:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766894.648271:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766894.648274:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098989056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766895.543778:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766896.567685:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766897.591639:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766898.615605:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766899.639784:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766899.767701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766899.767716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766899.767723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766899.767737:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4361 00000100:00080000:0.0:1775766899.767755:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766899.767765:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766899.767827:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098989184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766899.767873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766899.767879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766899.767881:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766899.767886:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4361 00000100:00080000:0.0:1775766899.767902:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766899.767907:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766899.767923:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766899.767948:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098989312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766899.767950:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098989184:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766899.767979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766899.767984:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766899.767986:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766899.767994:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4361 00000100:00080000:0.0:1775766899.767999:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766899.768004:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766899.768010:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766899.768026:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098989440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766899.768039:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766899.768044:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766899.768057:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766899.768063:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4361 00000100:00080000:0.0:1775766899.768068:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766899.768072:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766899.768079:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766899.768084:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766899.768103:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098989568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766899.768117:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766899.768121:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766899.768124:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766899.768130:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4361 00000100:00080000:0.0:1775766899.768135:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766899.768140:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766899.768149:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766899.768156:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766899/real 1775766899] req@ffff8b2382daf480 x1862022098989184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766899.768176:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098989696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766899.768189:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098989184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766899.768208:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766899.768214:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766899.768221:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098989184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766899.768257:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098989312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766899.768286:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766899.768301:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766899/real 1775766899] req@ffff8b2382daed80 x1862022098989312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766899.768315:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098989312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766899.768334:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766899.768339:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766899.768343:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098989312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766899.768350:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098989440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766899.768370:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766899.768380:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766899/real 1775766899] req@ffff8b2382daea00 x1862022098989440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766899.768393:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098989440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766899.768403:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766899.768407:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766899.768410:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098989440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766899.768417:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098989568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766899.768436:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766899.768446:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766899/real 1775766899] req@ffff8b2382dad500 x1862022098989568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766899.768504:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098989568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766899.768515:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766899.768518:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766899.768522:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098989568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766899.768529:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098989696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766899.768582:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766899.768598:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766899/real 1775766899] req@ffff8b2382dae680 x1862022098989696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766899.768612:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098989696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766899.768623:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766899.768627:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766899.768631:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098989696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766900.663799:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766901.687602:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766902.711700:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766903.735728:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766904.759754:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766904.887585:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766904.887602:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766904.887615:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766904.887629:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4366 00000100:00080000:1.0:1775766904.887645:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766904.887654:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766904.887707:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022098989824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766904.887741:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766904.887745:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766904.887746:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766904.887752:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4366 00000100:00080000:1.0:1775766904.887757:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766904.887763:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766904.887776:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766904.887780:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098989824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775766904.887797:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022098989952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766904.887808:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766904.887811:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766904.887813:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766904.887824:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4366 00000100:00080000:1.0:1775766904.887828:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766904.887831:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766904.887836:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766904.887849:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022098990080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766904.887856:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766904.887859:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766904.887861:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766904.887864:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4366 00000100:00080000:1.0:1775766904.887867:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766904.887869:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766904.887872:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766904.887878:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766904.887882:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022098990208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766904.887889:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766904.887893:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766904.887894:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766904.887897:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4366 00000100:00080000:1.0:1775766904.887900:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766904.887903:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766904.887905:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766904.887912:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022098990336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766904.887942:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766904/real 1775766904] req@ffff8b23864ffb80 x1862022098989824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766904.887961:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022098989824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766904.887976:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766904.887980:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766904.887985:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098989824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766904.888011:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098989952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766904.888031:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766904.888041:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766904/real 1775766904] req@ffff8b23864fe300 x1862022098989952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766904.888049:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022098989952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766904.888055:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766904.888057:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766904.888059:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098989952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766904.888063:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098990080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766904.888073:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766904.888079:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766904/real 1775766904] req@ffff8b23864ff100 x1862022098990080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766904.888086:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022098990080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766904.888092:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766904.888094:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766904.888096:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098990080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766904.888100:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098990208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766904.888111:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766904.888116:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766904/real 1775766904] req@ffff8b23864fd500 x1862022098990208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766904.888123:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022098990208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766904.888128:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766904.888130:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766904.888132:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098990208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766904.888135:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098990336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766904.888145:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766904.888150:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766904/real 1775766904] req@ffff8b23864fdc00 x1862022098990336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766904.888156:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022098990336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766904.888161:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766904.888163:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766904.888165:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098990336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766905.783666:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766906.807696:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766907.831777:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766908.855773:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766909.879736:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766910.007677:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766910.007693:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766910.007700:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766910.007716:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4371 00000100:00080000:3.0:1775766910.007735:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766910.007745:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766910.007806:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098990464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766910.007853:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766910.007859:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766910.007861:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766910.007867:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4371 00000100:00080000:3.0:1775766910.007871:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766910.007875:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766910.007888:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766910.007902:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098990592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766910.007917:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766910.007921:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766910.007924:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766910.007929:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4371 00000100:00080000:3.0:1775766910.007932:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766910.007937:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766910.007944:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766910.007956:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098990720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766910.007979:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766910.007988:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1775766910.007990:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098990464:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766910.007991:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766910.008007:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4371 00000100:00080000:3.0:1775766910.008013:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766910.008018:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766910.008028:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766910.008053:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098990848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766910.008072:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766910.008078:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766910.008083:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766910.008091:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4371 00000100:00080000:3.0:1775766910.008099:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775766910.008108:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766910.008116:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766910.008128:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766910.008155:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098990976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766910.008180:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766910/real 1775766910] req@ffff8b23864f0380 x1862022098990464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766910.008214:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098990464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766910.008231:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766910.008237:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766910.008243:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098990464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766910.008252:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098990592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766910.008280:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766910.008295:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766910/real 1775766910] req@ffff8b23864f2680 x1862022098990592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766910.008319:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098990592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766910.008329:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766910.008333:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766910.008337:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098990592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766910.008368:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098990720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766910.008389:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766910.008400:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766910/real 1775766910] req@ffff8b23864f0a80 x1862022098990720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766910.008415:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098990720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766910.008425:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766910.008428:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766910.008432:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098990720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766910.008438:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098990848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766910.008539:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766910.008555:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766910/real 1775766910] req@ffff8b23864f1f80 x1862022098990848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766910.008570:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098990848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766910.008588:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766910.008593:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766910.008597:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098990848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766910.008604:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098990976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766910.008628:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766910.008639:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766910/real 1775766910] req@ffff8b23864f1c00 x1862022098990976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766910.008652:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098990976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766910.008662:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766910.008665:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766910.008669:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098990976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766910.903657:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766911.927622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766912.951705:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766913.975650:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766914.999658:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766915.127596:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766915.127613:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766915.127621:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766915.127645:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4376 00000100:00080000:3.0:1775766915.127663:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766915.127670:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766915.127734:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098991104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766915.127783:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766915.127789:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766915.127792:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766915.127797:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4376 00000100:00080000:3.0:1775766915.127802:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766915.127807:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766915.127832:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766915.127853:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098991232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766915.127867:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766915.127871:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766915.127873:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766915.127878:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4376 00000100:00080000:3.0:1775766915.127881:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766915.127886:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766915.127891:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766915.127905:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098991360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766915.127920:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098991104:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766915.127920:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766915.127938:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766915.127941:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766915.127946:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4376 00000100:00080000:3.0:1775766915.127949:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766915.127952:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766915.127956:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766915.127967:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098991488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766915.127976:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766915.127979:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766915.127981:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766915.127985:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4376 00000100:00080000:3.0:1775766915.127991:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766915.127994:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766915.128014:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766915.128023:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098991616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766915.128092:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098991232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766915.128131:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098991360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766915.128166:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098991488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766915.128190:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098991616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766916.023669:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766917.047638:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766918.071671:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775766918.199698:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775766918.199737:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766918.199751:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766918.199799:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766918.199817:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766918.199827:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775766918.199836:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775766918.199943:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766915/real 1775766918] req@ffff8b23864f0700 x1862022098991360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766918.199974:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098991360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766918.199989:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766918.199994:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766918.199999:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098991360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766918.200008:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766915/real 1775766918] req@ffff8b23864f3100 x1862022098991488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766918.200021:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098991488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766918.200029:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766918.200033:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766918.200037:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098991488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766918.200044:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766915/real 1775766918] req@ffff8b23864f2d80 x1862022098991616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766918.200056:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098991616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766918.200064:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766918.200067:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766918.200072:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098991616:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766918.200079:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766915/real 1775766918] req@ffff8b23864f3b80 x1862022098991104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766918.200091:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098991104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766918.200098:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766918.200102:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766918.200105:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098991104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766918.200112:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766915/real 1775766918] req@ffff8b23864f1180 x1862022098991232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766918.200123:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098991232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766918.200130:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766918.200133:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766918.200137:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098991232:192.168.202.128@tcp:38:kworker.0 00010000:00010000:0.0:1775766919.095595:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766920.119607:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766920.247650:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766920.247659:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766920.247662:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766920.247682:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4381 00000100:00080000:0.0:1775766920.247692:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766920.247698:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766920.247744:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098991744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766920.247776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766920.247780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766920.247782:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766920.247786:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4381 00000100:00080000:0.0:1775766920.247790:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766920.247794:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766920.247805:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766920.247821:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098991872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766920.247835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766920.247838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766920.247842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766920.247850:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4381 00000100:00100000:3.0:1775766920.247850:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098991744:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766920.247855:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766920.247867:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766920.247874:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766920.247885:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098992000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766920.247892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766920.247895:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766920.247896:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766920.247900:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4381 00000100:00080000:0.0:1775766920.247902:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766920.247905:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766920.247908:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766920.247917:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098992128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766920.247923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766920.247926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766920.247927:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766920.247929:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4381 00000100:00080000:0.0:1775766920.247932:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766920.247936:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766920.247938:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766920.247945:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098992256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775766920.247975:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766920.248061:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766920/real 1775766920] req@ffff8b2382dac380 x1862022098991744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766920.248094:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098991744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766920.248118:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766920.248125:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766920.248131:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098991744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766920.248172:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098991872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766920.248205:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766920.248222:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766920/real 1775766920] req@ffff8b2382dad880 x1862022098991872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766920.248238:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098991872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766920.248248:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766920.248252:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766920.248256:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098991872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766920.248264:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098992000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766920.248292:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766920.248306:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766920/real 1775766920] req@ffff8b2382daca80 x1862022098992000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766920.248321:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098992000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766920.248331:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766920.248335:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766920.248355:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098992000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766920.248366:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098992128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766920.248395:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766920.248409:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766920/real 1775766920] req@ffff8b2382daf100 x1862022098992128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766920.248425:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098992128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766920.248434:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766920.248438:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766920.248442:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098992128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766920.248450:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098992256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766920.248512:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766920.248668:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766920/real 1775766920] req@ffff8b2382dac000 x1862022098992256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766920.248684:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098992256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766920.248700:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766920.248705:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766920.248709:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098992256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766921.143658:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766922.167731:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766923.191570:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766924.215668:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766925.239586:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766925.367588:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766925.367599:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766925.367604:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766925.367615:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4386 00000100:00080000:0.0:1775766925.367627:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766925.367635:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766925.367679:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022098992384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766925.367706:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766925.367710:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766925.367712:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766925.367718:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4386 00000100:00080000:0.0:1775766925.367723:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766925.367728:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766925.367734:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098992384:192.168.202.128@tcp:250:kworker.0 00000008:00100000:0.0:1775766925.367739:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766925.367757:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022098992512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766925.367770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766925.367774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766925.367776:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766925.367781:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4386 00000100:00080000:0.0:1775766925.367786:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766925.367793:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775766925.367794:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:0.0:1775766925.367802:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766925.367817:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022098992640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766925.367829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766925.367832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766925.367834:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775766925.367840:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766925/real 1775766925] req@ffff8b2382dac000 x1862022098992384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766925.367841:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4386 00000100:00080000:0.0:1775766925.367855:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766925.367861:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766925.367866:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022098992384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775766925.367868:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775766925.367893:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766925.367898:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766925.367899:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022098992768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766925.367902:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098992384:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766925.367917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766925.367921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766925.367923:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766925.367928:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4386 00000100:00100000:3.0:1775766925.367931:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098992512:192.168.202.128@tcp:38:kworker.0 00000100:00080000:0.0:1775766925.367934:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766925.367940:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766925.367946:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766925.367955:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766925.367964:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022098992896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766925.367968:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766925/real 1775766925] req@ffff8b2382daf100 x1862022098992512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766925.367992:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022098992512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766925.367998:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766925.368001:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766925.368004:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098992512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766925.368009:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098992640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766925.368022:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766925.368030:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766925/real 1775766925] req@ffff8b2382daca80 x1862022098992640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766925.368057:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022098992640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766925.368068:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766925.368071:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766925.368073:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098992640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766925.368076:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098992768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766925.368089:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766925.368096:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766925/real 1775766925] req@ffff8b2382dad880 x1862022098992768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766925.368103:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022098992768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766925.368109:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766925.368111:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766925.368113:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098992768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766925.368125:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098992896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766925.368136:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766925.368142:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766925/real 1775766925] req@ffff8b2382dac380 x1862022098992896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766925.368166:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022098992896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766925.368171:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766925.368172:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766925.368175:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098992896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766926.263642:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766927.287671:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766928.311660:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766929.335658:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766930.359650:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766930.487613:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766930.487622:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766930.487625:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766930.487651:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4391 00000100:00080000:3.0:1775766930.487661:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766930.487666:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766930.487733:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098993024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766930.487761:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766930.487764:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766930.487766:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766930.487792:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4391 00000100:00080000:3.0:1775766930.487796:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766930.487803:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766930.487814:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766930.487825:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098993024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766930.487836:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098993152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766930.487851:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766930.487856:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766930.487858:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766930.487863:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4391 00000100:00080000:3.0:1775766930.487867:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766930.487872:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766930.487878:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766930.487892:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098993280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766930.487903:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766930.487907:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766930.487909:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766930.487913:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4391 00000100:00080000:3.0:1775766930.487916:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766930.487919:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766930.487927:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766930.487928:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766930.487950:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098993408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766930.487962:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766930.487966:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766930.487969:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766930.487976:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4391 00000100:00080000:3.0:1775766930.487998:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766930.488001:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766930/real 1775766930] req@ffff8b23864f0000 x1862022098993024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766930.488002:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766930.488012:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766930.488025:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098993536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766930.488040:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098993024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766930.488056:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766930.488063:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766930.488068:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098993024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766930.488095:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098993152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766930.488114:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766930.488123:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766930/real 1775766930] req@ffff8b23864f2a00 x1862022098993152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766930.488138:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098993152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766930.488144:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766930.488146:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766930.488149:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098993152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766930.488153:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098993280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766930.488168:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766930.488185:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766930/real 1775766930] req@ffff8b23864f3480 x1862022098993280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766930.488193:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098993280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766930.488199:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766930.488201:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766930.488203:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098993280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766930.488208:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098993408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766930.488221:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766930.488227:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766930/real 1775766930] req@ffff8b23864f1180 x1862022098993408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766930.488249:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098993408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766930.488255:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766930.488258:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766930.488260:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098993408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766930.488264:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098993536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766930.488277:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766930.488283:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766930/real 1775766930] req@ffff8b23864f3b80 x1862022098993536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766930.488290:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098993536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766930.488295:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766930.488297:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766930.488299:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098993536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766931.383663:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766932.407656:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766933.431637:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766934.455629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766935.479622:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766935.607521:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766935.607529:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766935.607533:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766935.607542:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4397 00000100:00080000:3.0:1775766935.607552:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766935.607557:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766935.607603:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098993664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766935.607625:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766935.607628:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766935.607629:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766935.607631:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4397 00000100:00080000:3.0:1775766935.607634:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766935.607636:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766935.607644:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766935.607653:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098993792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766935.607662:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766935.607664:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766935.607666:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766935.607671:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4397 00000100:00100000:0.0:1775766935.607673:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098993664:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766935.607673:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766935.607676:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766935.607682:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766935.607689:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098993920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766935.607696:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766935.607698:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766935.607699:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766935.607702:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4397 00000100:00080000:3.0:1775766935.607704:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766935.607706:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766935.607709:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766935.607716:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098994048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766935.607721:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766935.607723:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766935.607725:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766935.607727:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4397 00000100:00080000:3.0:1775766935.607730:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766935.607731:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766935.607734:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766935.607738:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766935.607748:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098994176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766935.607773:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766935/real 1775766935] req@ffff8b23864f2d80 x1862022098993664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766935.607788:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098993664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766935.607796:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766935.607799:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766935.607802:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098993664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766935.607820:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098993792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766935.607835:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766935.607843:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766935/real 1775766935] req@ffff8b23864f3100 x1862022098993792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766935.607851:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098993792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766935.607856:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766935.607858:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766935.607860:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098993792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766935.607864:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098993920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766935.607875:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766935.607880:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766935/real 1775766935] req@ffff8b23864f0700 x1862022098993920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766935.607892:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098993920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766935.607897:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766935.607899:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766935.607901:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098993920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766935.607904:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098994048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766935.607916:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766935.607921:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766935/real 1775766935] req@ffff8b23864f1c00 x1862022098994048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766935.607929:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098994048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766935.607933:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766935.607935:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766935.607937:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098994048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766935.607940:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098994176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766935.607949:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766935.607954:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766935/real 1775766935] req@ffff8b23864f1f80 x1862022098994176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766935.607961:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098994176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766935.607971:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766935.607973:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766935.607975:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098994176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766936.503661:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766937.527616:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766938.551717:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766939.575674:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766940.599667:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766940.727618:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766940.727632:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766940.727639:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766940.727653:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4402 00000100:00080000:0.0:1775766940.727669:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766940.727677:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766940.727735:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022098994304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766940.727777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766940.727782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766940.727784:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766940.727788:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4402 00000100:00080000:0.0:1775766940.727791:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766940.727795:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766940.727808:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766940.727823:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022098994432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766940.727840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775766940.727843:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098994304:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766940.727869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766940.727872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766940.727878:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4402 00000100:00080000:0.0:1775766940.727881:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766940.727884:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766940.727890:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766940.727905:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022098994560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766940.727915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766940.727918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766940.727920:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766940.727924:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4402 00000100:00080000:0.0:1775766940.727926:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766940.727931:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766940.727935:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766940.727944:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022098994688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766940.727954:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766940.727958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766940.727960:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775766940.727964:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775766940.727969:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4402 00000100:00080000:0.0:1775766940.727973:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766940.727977:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766940.727985:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766940.728003:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098994816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766940.728041:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766940/real 1775766940] req@ffff8b2382dae680 x1862022098994304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766940.728074:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022098994304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766940.728087:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766940.728092:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766940.728096:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098994304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766940.728162:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098994432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766940.728200:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766940.728215:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766940/real 1775766940] req@ffff8b2382dad500 x1862022098994432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766940.728228:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022098994432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766940.728236:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766940.728240:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766940.728243:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098994432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766940.728249:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098994560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766940.728268:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766940.728277:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766940/real 1775766940] req@ffff8b2382daea00 x1862022098994560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766940.728289:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022098994560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766940.728303:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766940.728307:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766940.728310:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098994560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766940.728316:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098994688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766940.728332:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766940.728341:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766940/real 1775766940] req@ffff8b2382daed80 x1862022098994688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766940.728352:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022098994688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766940.728360:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766940.728363:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766940.728365:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098994688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766940.728371:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098994816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766940.728386:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766940.728394:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766940/real 1775766940] req@ffff8b2382daf480 x1862022098994816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766940.728404:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098994816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766940.728412:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766940.728415:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766940.728419:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098994816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766941.623662:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766942.647629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766943.671713:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766944.695688:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766945.719641:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766945.847512:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766945.847519:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766945.847523:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766945.847532:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4407 00000100:00080000:0.0:1775766945.847541:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766945.847562:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775766945.847599:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022098994944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766945.847625:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766945.847628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766945.847630:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766945.847633:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4407 00000100:00080000:0.0:1775766945.847635:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766945.847638:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766945.847646:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766945.847657:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022098995072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766945.847667:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766945.847671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766945.847672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766945.847678:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4407 00000100:00100000:3.0:1775766945.847680:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098994944:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775766945.847681:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766945.847693:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766945.847698:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766945.847710:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022098995200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766945.847719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766945.847722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766945.847724:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766945.847727:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4407 00000100:00080000:0.0:1775766945.847733:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766945.847735:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766945.847739:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775766945.847744:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766945.847752:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022098995328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766945.847760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766945.847763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775766945.847764:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766945.847767:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4407 00000100:00080000:0.0:1775766945.847769:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766945.847772:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775766945.847776:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766945.847786:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022098995456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775766945.847795:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766945/real 1775766945] req@ffff8b2382dac700 x1862022098994944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766945.847837:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022098994944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766945.847849:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766945.847853:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766945.847858:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098994944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775766945.847866:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098995072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766945.847893:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766945.847904:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766945/real 1775766945] req@ffff8b2382dace00 x1862022098995072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766945.847914:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022098995072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766945.847920:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766945.847922:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766945.847925:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098995072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766945.847946:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098995200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775766945.847959:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766945.847967:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766945/real 1775766945] req@ffff8b2382dadf80 x1862022098995200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766945.847974:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022098995200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766945.847979:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766945.847981:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766945.847983:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098995200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775766945.847987:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098995328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766945.847997:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766945.848003:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766945/real 1775766945] req@ffff8b2382dafb80 x1862022098995328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766945.848010:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022098995328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766945.848015:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766945.848016:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766945.848018:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098995328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775766945.848022:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098995456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775766945.848032:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766945.848037:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766945/real 1775766945] req@ffff8b2382daf480 x1862022098995456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766945.848044:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022098995456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766945.848049:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775766945.848051:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775766945.848053:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098995456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766946.743609:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766947.767628:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766948.791608:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766949.815669:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766950.839621:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766950.967562:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766950.967570:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766950.967573:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766950.967582:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4412 00000100:00080000:3.0:1775766950.967591:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766950.967596:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766950.967631:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098995584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766950.967656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766950.967659:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766950.967660:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766950.967663:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4412 00000100:00080000:3.0:1775766950.967665:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766950.967668:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766950.967675:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766950.967684:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098995712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766950.967693:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766950.967697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766950.967699:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766950.967700:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098995584:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766950.967705:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4412 00000100:00080000:3.0:1775766950.967708:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766950.967710:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766950.967714:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766950.967722:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098995840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766950.967728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766950.967731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766950.967732:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766950.967735:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4412 00000100:00080000:3.0:1775766950.967736:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766950.967738:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766950.967741:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766950.967749:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098995968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766950.967755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766950.967757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766950.967758:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766950.967760:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4412 00000100:00080000:3.0:1775766950.967763:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775766950.967767:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775766950.967770:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766950.967774:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766950.967783:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098996096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766950.967807:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766950/real 1775766950] req@ffff8b23864f0a80 x1862022098995584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766950.967821:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098995584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766950.967828:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766950.967831:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766950.967834:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098995584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766950.967852:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098995712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766950.967867:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766950.967874:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766950/real 1775766950] req@ffff8b23864f2680 x1862022098995712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766950.967882:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098995712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766950.967887:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766950.967890:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766950.967892:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098995712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766950.967896:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098995840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766950.967918:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766950.967923:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766950/real 1775766950] req@ffff8b23864f0380 x1862022098995840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766950.967930:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098995840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766950.967935:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766950.967937:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766950.967939:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098995840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766950.967943:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098995968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766950.967953:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766950.967958:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766950/real 1775766950] req@ffff8b23864f1f80 x1862022098995968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766950.967965:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098995968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766950.967981:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766950.967983:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766950.967985:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098995968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766950.967988:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098996096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766950.967998:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766950.968012:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766950/real 1775766950] req@ffff8b23864f1c00 x1862022098996096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766950.968033:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098996096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766950.968039:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766950.968041:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766950.968043:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098996096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766951.863637:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766952.887679:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766953.911659:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766954.936686:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766955.959677:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766956.087603:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766956.087627:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766956.087633:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766956.087644:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4417 00000100:00080000:1.0:1775766956.087657:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766956.087734:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766956.087777:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022098996224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766956.087807:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766956.087811:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766956.087812:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766956.087815:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4417 00000100:00080000:1.0:1775766956.087818:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766956.087821:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766956.087830:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766956.087842:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022098996352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766956.087851:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766956.087854:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766956.087855:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766956.087863:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098996224:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766956.087870:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4417 00000100:00080000:1.0:1775766956.087873:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766956.087877:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766956.087880:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766956.087891:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022098996480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766956.087898:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766956.087908:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766956.087910:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766956.087913:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4417 00000100:00080000:1.0:1775766956.087922:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766956.087925:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766956.087929:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766956.087934:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775766956.087939:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022098996608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766956.087945:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766956.087948:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766956.087949:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766956.087952:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4417 00000100:00080000:1.0:1775766956.087955:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766956.087958:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766956.087960:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766956.087982:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022098996736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766956.088002:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766956/real 1775766956] req@ffff8b23864fe680 x1862022098996224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766956.088022:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022098996224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766956.088031:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766956.088035:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766956.088038:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098996224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766956.088044:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098996352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766956.088062:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766956.089153:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766956/real 1775766956] req@ffff8b23864fca80 x1862022098996352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766956.089169:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022098996352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766956.089175:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766956.089178:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766956.089180:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098996352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766956.089205:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098996480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766956.089223:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766956.089231:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766956/real 1775766956] req@ffff8b23864fea00 x1862022098996480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766956.089239:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022098996480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766956.089244:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766956.089246:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766956.089248:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098996480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766956.089251:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098996608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766956.089263:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766956.089268:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766956/real 1775766956] req@ffff8b23864fd180 x1862022098996608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766956.089275:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022098996608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766956.089285:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766956.089288:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766956.089290:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098996608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766956.089293:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098996736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766956.089302:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766956.089307:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766956/real 1775766956] req@ffff8b23864fce00 x1862022098996736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766956.089314:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022098996736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766956.089320:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766956.089322:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766956.089323:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098996736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766956.983591:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766958.007777:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766959.031678:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766960.055695:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766961.079649:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766961.207570:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766961.207579:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766961.207583:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766961.207593:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4422 00000100:00080000:3.0:1775766961.207604:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766961.207611:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766961.207651:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098996864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766961.207679:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766961.207683:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766961.207685:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766961.207688:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4422 00000100:00080000:3.0:1775766961.207691:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766961.207693:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766961.207701:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766961.207713:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098996992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766961.207725:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766961.207729:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766961.207730:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766961.207733:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4422 00000100:00080000:3.0:1775766961.207735:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766961.207740:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766961.207744:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766961.207754:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098997120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766961.207775:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098996864:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766961.207780:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766961.207793:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766961.207795:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766961.207799:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4422 00000100:00080000:3.0:1775766961.207801:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766961.207804:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766961.207809:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766961.207822:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098997248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766961.207834:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766961.207838:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766961.207841:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766961.207845:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4422 00000100:00080000:3.0:1775766961.207850:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766961.207855:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766961.207862:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766961.207876:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098997376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766961.207899:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766961.207993:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766961/real 1775766961] req@ffff8b23864f0700 x1862022098996864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766961.208011:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098996864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766961.208021:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766961.208025:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766961.208029:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098996864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766961.208036:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098996992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766961.208056:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766961.208066:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766961/real 1775766961] req@ffff8b23864f3100 x1862022098996992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766961.208078:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098996992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766961.208087:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766961.208090:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766961.208094:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098996992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766961.208126:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098997120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766961.208147:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766961.208156:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766961/real 1775766961] req@ffff8b23864f2d80 x1862022098997120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766961.208165:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098997120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766961.208172:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766961.208174:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766961.208176:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098997120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766961.208181:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098997248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766961.208247:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766961.208254:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766961/real 1775766961] req@ffff8b23864f3b80 x1862022098997248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766961.208263:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098997248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766961.208268:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766961.208271:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766961.208273:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098997248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766961.208277:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098997376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766961.208291:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766961.208296:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766961/real 1775766961] req@ffff8b23864f1180 x1862022098997376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766961.208304:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098997376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766961.208309:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766961.208311:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766961.208313:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098997376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766962.103632:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766963.127657:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766964.151706:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766965.175744:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766966.199606:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766966.327586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766966.327594:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766966.327597:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766966.327606:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4427 00000100:00080000:3.0:1775766966.327616:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766966.327623:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766966.327668:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022098997504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766966.327697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766966.327701:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766966.327703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766966.327707:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4427 00000100:00080000:3.0:1775766966.327711:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766966.327715:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766966.327723:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766966.327736:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022098997632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766966.327745:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098997504:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766966.327747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766966.327754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766966.327755:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766966.327759:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4427 00000100:00080000:3.0:1775766966.327761:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766966.327764:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766966.327767:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766966.327777:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022098997760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766966.327785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766966.327787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766966.327788:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766966.327791:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4427 00000100:00080000:3.0:1775766966.327793:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766966.327795:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766966.327798:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766966.327802:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766966.327809:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022098997888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766966.327817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766966.327820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766966.327821:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766966.327823:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4427 00000100:00080000:3.0:1775766966.327825:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766966.327827:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766966.327831:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766966.327839:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766966/real 1775766966] req@ffff8b23864f3480 x1862022098997504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766966.327841:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022098998016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766966.327886:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022098997504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766966.327894:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766966.327898:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766966.327906:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098997504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766966.327925:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098997632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766966.327942:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766966.327950:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766966/real 1775766966] req@ffff8b23864f2a00 x1862022098997632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766966.327959:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022098997632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766966.327964:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766966.327966:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766966.327969:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098997632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766966.327972:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098997760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766966.327982:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766966.327988:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766966/real 1775766966] req@ffff8b23864f0000 x1862022098997760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766966.327995:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022098997760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766966.328005:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766966.328007:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766966.328009:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098997760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766966.328013:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098997888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766966.328025:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766966.328034:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766966/real 1775766966] req@ffff8b23864f1180 x1862022098997888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766966.328047:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022098997888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766966.328055:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766966.328059:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766966.328062:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098997888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766966.328069:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098998016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766966.328088:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766966.328097:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766966/real 1775766966] req@ffff8b23864f3b80 x1862022098998016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766966.328115:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022098998016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766966.328120:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766966.328123:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766966.328125:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098998016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766967.223742:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766968.247698:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766969.271752:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766970.295627:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766971.319638:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766971.447567:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766971.447579:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766971.447584:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766971.447595:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4432 00000100:00080000:3.0:1775766971.447610:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766971.447617:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766971.447665:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022098998144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766971.447697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766971.447701:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766971.447703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766971.447706:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4432 00000100:00080000:3.0:1775766971.447709:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766971.447712:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766971.447721:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766971.447732:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022098998272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766971.447743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766971.447746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766971.447748:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766971.447752:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4432 00000100:00080000:3.0:1775766971.447755:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766971.447758:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766971.447761:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766971.447769:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022098998400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766971.447779:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098998144:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766971.447780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766971.447789:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766971.447793:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766971.447800:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4432 00000100:00080000:3.0:1775766971.447816:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766971.447844:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766971.447850:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766971.447868:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098998528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766971.447880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766971.447883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766971.447886:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766971.447890:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4432 00000100:00080000:3.0:1775766971.447894:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766971.447897:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766971.447901:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766971.447903:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766971.447913:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098998656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766971.447961:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766971/real 1775766971] req@ffff8b23864f2d80 x1862022098998144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766971.447984:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022098998144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766971.447996:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766971.448001:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766971.448006:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098998144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766971.448015:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098998272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766971.448040:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766971.448055:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766971/real 1775766971] req@ffff8b23864f3100 x1862022098998272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766971.448068:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022098998272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766971.448077:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766971.448081:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766971.448085:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098998272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766971.448134:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098998400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766971.448165:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766971.448180:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766971/real 1775766971] req@ffff8b23864f0700 x1862022098998400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766971.448198:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022098998400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766971.448209:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766971.448214:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766971.448219:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098998400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766971.448227:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098998528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766971.448266:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766971.448278:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766971/real 1775766971] req@ffff8b23864f1c00 x1862022098998528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766971.448291:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098998528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766971.448301:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766971.448320:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766971.448324:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098998528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766971.448330:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098998656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766971.448350:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766971.448360:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766971/real 1775766971] req@ffff8b23864f1f80 x1862022098998656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766971.448378:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098998656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766971.448386:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766971.448389:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766971.448393:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098998656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766972.343638:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766973.367689:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766974.391588:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766975.415677:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766976.439668:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766976.567646:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766976.567656:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766976.567662:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766976.567672:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4437 00000100:00080000:1.0:1775766976.567684:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766976.567691:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766976.567736:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022098998784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766976.567791:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766976.567796:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766976.567798:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766976.567802:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4437 00000100:00080000:1.0:1775766976.567806:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766976.567810:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766976.567821:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766976.567844:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022098998912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766976.567864:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766976.567867:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098998784:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766976.567874:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766976.567879:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766976.567887:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4437 00000100:00080000:1.0:1775766976.567895:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766976.567902:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766976.567911:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766976.567932:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022098999040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766976.567943:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766976.567946:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766976.567949:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775766976.567958:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775766976.567961:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4437 00000100:00080000:1.0:1775766976.567968:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766976.567972:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766976.567978:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766976.567990:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022098999168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766976.567999:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766976.568002:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766976.568004:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766976.568008:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4437 00000100:00080000:1.0:1775766976.568012:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766976.568015:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766976.568020:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766976.568022:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766976/real 1775766976] req@ffff8b23864fdf80 x1862022098998784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766976.568036:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022098999296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766976.568051:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022098998784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766976.568064:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766976.568069:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766976.568073:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098998784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766976.568100:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098998912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766976.568125:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766976.568137:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766976/real 1775766976] req@ffff8b23864ff800 x1862022098998912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766976.568155:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022098998912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766976.568163:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766976.568165:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766976.568168:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098998912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766976.568174:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098999040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766976.568189:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766976.568198:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766976/real 1775766976] req@ffff8b23864fed80 x1862022098999040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766976.568208:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022098999040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766976.568215:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766976.568218:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766976.568221:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098999040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766976.568225:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098999168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766976.568240:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766976.568248:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766976/real 1775766976] req@ffff8b23864fc000 x1862022098999168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766976.568258:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022098999168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766976.568271:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766976.568274:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766976.568277:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098999168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766976.568282:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098999296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766976.568296:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766976.568303:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766976/real 1775766976] req@ffff8b23864fc380 x1862022098999296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766976.568313:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022098999296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766976.568320:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766976.568323:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766976.568325:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098999296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766977.463593:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766978.487679:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766979.511644:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766980.535636:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766981.559661:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766981.687793:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766981.687804:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766981.687810:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766981.687820:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4443 00000100:00080000:3.0:1775766981.687832:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766981.687839:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766981.687883:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022098999424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766981.687913:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766981.687917:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766981.687919:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766981.687922:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4443 00000100:00080000:3.0:1775766981.687924:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766981.687927:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766981.687937:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766981.687949:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022098999552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766981.687959:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766981.687980:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766981.687982:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766981.687986:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4443 00000100:00080000:3.0:1775766981.687990:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766981.687997:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098999424:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766981.687999:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766981.688009:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766981.688021:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022098999680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766981.688032:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766981.688036:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766981.688037:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766981.688041:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4443 00000100:00080000:3.0:1775766981.688046:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766981.688050:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766981.688054:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766981.688066:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022098999808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766981.688075:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766981.688078:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766981.688080:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766981.688083:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4443 00000100:00080000:3.0:1775766981.688087:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766981.688090:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766981.688111:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766981.688124:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022098999936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766981.688188:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098999552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766981.688248:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098999680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766981.688287:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098999808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766981.688306:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098999936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766982.583622:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766983.607732:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766984.631636:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775766984.759806:0:2363:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775766984.759858:0:2363:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775766984.759881:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766984.759951:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766984.759962:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766984.759972:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775766984.759984:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775766984.760033:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766981/real 1775766984] req@ffff8b23864f0a80 x1862022098999680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766984.760069:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022098999680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766984.760084:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766984.760090:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766984.760095:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098999680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766984.760104:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766981/real 1775766984] req@ffff8b23864f1f80 x1862022098999808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766984.760129:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022098999808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766984.760138:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766984.760142:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766984.760146:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098999808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766984.760155:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766981/real 1775766984] req@ffff8b23864f1c00 x1862022098999936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766984.760170:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022098999936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766984.760180:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766984.760186:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766984.760189:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022098999936:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766984.760198:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766981/real 1775766984] req@ffff8b23864f0380 x1862022098999424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766984.760215:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022098999424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766984.760231:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766984.760237:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766984.760241:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022098999424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766984.760251:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766981/real 1775766984] req@ffff8b23864f2680 x1862022098999552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766984.760267:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022098999552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766984.760276:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766984.760281:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766984.760285:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022098999552:192.168.202.128@tcp:38:kworker.0 00010000:00010000:1.0:1775766985.655699:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766986.679673:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766986.807576:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766986.807613:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766986.807618:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766986.807640:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4448 00000100:00080000:3.0:1775766986.807687:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766986.807708:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766986.807757:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099000064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766986.807796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766986.807804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766986.807806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766986.807812:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4448 00000100:00080000:3.0:1775766986.807817:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766986.807821:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766986.807834:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766986.807850:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099000192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766986.807866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766986.807869:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099000064:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766986.807873:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766986.807877:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766986.807885:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4448 00000100:00080000:3.0:1775766986.807890:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766986.807894:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766986.807900:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766986.807916:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099000320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766986.807927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766986.807931:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766986.807933:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766986.807938:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4448 00000100:00080000:3.0:1775766986.807943:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766986.807948:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766986.807956:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766986.807963:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775766986.807970:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099000448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766986.807981:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766986.807985:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766986.807989:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766986.807993:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4448 00000100:00080000:3.0:1775766986.807998:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766986.808003:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766986.808009:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766986.808023:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099000576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766986.808042:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766986/real 1775766986] req@ffff8b23864f0700 x1862022099000064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766986.808068:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099000064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766986.808084:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766986.808089:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766986.808095:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099000064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766986.808141:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099000192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766986.808170:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766986.808185:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766986/real 1775766986] req@ffff8b23864f3100 x1862022099000192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766986.808200:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099000192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766986.808211:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766986.808215:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766986.808219:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099000192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766986.808226:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099000320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766986.808250:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766986.808263:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766986/real 1775766986] req@ffff8b23864f2d80 x1862022099000320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766986.808275:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099000320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766986.808286:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766986.808290:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766986.808294:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099000320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766986.808302:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099000448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766986.808330:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766986.808342:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766986/real 1775766986] req@ffff8b23864f3b80 x1862022099000448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766986.808355:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099000448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766986.808363:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766986.808367:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766986.808371:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099000448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766986.808378:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099000576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766986.808401:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766986.808412:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766986/real 1775766986] req@ffff8b23864f1180 x1862022099000576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766986.808424:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099000576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766986.808433:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766986.808437:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766986.808441:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099000576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766987.703631:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766988.727656:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766989.751634:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766990.775691:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766991.799621:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766991.927567:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766991.927585:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766991.927602:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766991.927613:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4453 00000100:00080000:1.0:1775766991.927625:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766991.927632:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775766991.927673:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099000704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766991.927704:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766991.927709:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766991.927711:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766991.927716:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4453 00000100:00080000:1.0:1775766991.927720:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766991.927724:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766991.927742:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766991.927753:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099000704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775766991.927760:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099000832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766991.927776:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766991.927780:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766991.927782:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766991.927787:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4453 00000100:00080000:1.0:1775766991.927792:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766991.927796:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766991.927802:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766991.927816:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099000960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766991.927827:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775766991.927829:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766991.927833:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766991.927835:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766991.927842:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4453 00000100:00080000:1.0:1775766991.927847:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766991.927851:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766991.927857:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766991.927867:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766991/real 1775766991] req@ffff8b23864fc380 x1862022099000704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766991.927874:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099001088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766991.927887:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099000704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766991.927887:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766991.927898:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775766991.927902:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766991.927904:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775766991.927909:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4453 00000100:00080000:0.0:1775766991.927910:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766991.927916:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099000704:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775766991.927917:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766991.927925:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775766991.927930:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766991.927941:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099000832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:1.0:1775766991.927945:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099001216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766991.927965:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766991.927975:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766991/real 1775766991] req@ffff8b23864fc000 x1862022099000832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766991.927984:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099000832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766991.927990:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766991.927992:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766991.927994:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099000832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766991.927998:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099000960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766991.928009:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766991.928014:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766991/real 1775766991] req@ffff8b23864fed80 x1862022099000960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766991.928026:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099000960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766991.928031:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766991.928033:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766991.928044:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099000960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766991.928048:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099001088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766991.928060:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766991.928067:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766991/real 1775766991] req@ffff8b23864ff800 x1862022099001088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766991.928074:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099001088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766991.928079:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766991.928081:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766991.928083:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099001088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766991.928093:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099001216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766991.928103:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766991.928108:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766991/real 1775766991] req@ffff8b23864fdf80 x1862022099001216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766991.928115:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099001216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766991.928124:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766991.928127:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766991.928129:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099001216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775766992.823689:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766993.847695:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766994.871666:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766995.895723:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766996.919713:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766997.047619:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766997.047632:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766997.047666:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766997.047679:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4458 00000100:00080000:3.0:1775766997.047695:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766997.047703:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775766997.047758:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099001344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766997.047830:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766997.047836:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766997.047839:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766997.047845:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4458 00000100:00080000:3.0:1775766997.047852:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766997.047857:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766997.047871:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766997.047893:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099001472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766997.047910:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766997.047916:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099001344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775766997.047916:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766997.047920:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766997.047933:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4458 00000100:00080000:3.0:1775766997.047938:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766997.047943:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766997.047950:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766997.047967:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099001600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766997.047979:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766997.047984:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766997.047986:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766997.047991:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4458 00000100:00080000:3.0:1775766997.047996:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766997.048001:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775766997.048014:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:3.0:1775766997.048018:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766997.048038:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099001728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766997.048058:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766997.048063:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775766997.048065:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766997.048072:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4458 00000100:00080000:3.0:1775766997.048077:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766997.048083:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766997/real 1775766997] req@ffff8b23864f0000 x1862022099001344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766997.048085:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775766997.048111:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766997.048123:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099001344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766997.048131:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099001856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766997.048138:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766997.048177:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766997.048183:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099001344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775766997.048233:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099001472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766997.048268:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766997.048280:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766997/real 1775766997] req@ffff8b23864f2a00 x1862022099001472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766997.048291:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099001472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766997.048305:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766997.048308:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766997.048311:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099001472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766997.048316:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099001600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775766997.048331:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766997.048339:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766997/real 1775766997] req@ffff8b23864f3480 x1862022099001600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766997.048348:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099001600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766997.048365:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766997.048367:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766997.048370:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099001600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775766997.048374:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099001728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766997.048389:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766997.048397:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766997/real 1775766997] req@ffff8b23864f1180 x1862022099001728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766997.048406:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099001728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766997.048413:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766997.048415:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766997.048418:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099001728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775766997.048432:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099001856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775766997.048489:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775766997.048498:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766997/real 1775766997] req@ffff8b23864f3b80 x1862022099001856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766997.048511:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099001856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766997.048522:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766997.048527:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775766997.048531:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099001856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775766997.943654:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766998.967647:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766999.991709:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767001.015680:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767002.039662:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767002.167659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767002.167671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767002.167677:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767002.167689:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4463 00000100:00080000:1.0:1775767002.167703:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767002.167712:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767002.167762:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099001984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767002.167791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767002.167795:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767002.167796:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767002.167800:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4463 00000100:00080000:1.0:1775767002.167802:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767002.167805:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767002.167814:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767002.167826:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099002112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767002.167836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767002.167839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767002.167841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767002.167845:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4463 00000100:00080000:1.0:1775767002.167849:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767002.167852:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767002.167855:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767002.167867:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099002240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767002.167871:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099001984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767002.167884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767002.167887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767002.167889:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767002.167893:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4463 00000100:00080000:1.0:1775767002.167895:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767002.167897:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767002.167901:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767002.167914:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099002368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767002.167920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767002.167923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767002.167924:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767002.167926:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4463 00000100:00080000:1.0:1775767002.167928:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767002.167931:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767002.167934:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767002.167942:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099002496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767002.167975:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767002.168023:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767002/real 1775767002] req@ffff8b23864fce00 x1862022099001984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767002.168045:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099001984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767002.168059:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767002.168064:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767002.168069:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099001984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767002.168079:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099002112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767002.168108:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767002.168123:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767002/real 1775767002] req@ffff8b23864fd180 x1862022099002112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767002.168139:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099002112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767002.168150:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767002.168154:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767002.168159:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099002112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767002.168189:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099002240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767002.168215:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767002.168229:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767002/real 1775767002] req@ffff8b23864fea00 x1862022099002240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767002.168244:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099002240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767002.168255:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767002.168259:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767002.168263:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099002240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767002.168272:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099002368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767002.168296:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767002.168309:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767002/real 1775767002] req@ffff8b23864fca80 x1862022099002368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767002.168334:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099002368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767002.168344:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767002.168348:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767002.168352:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099002368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767002.168372:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099002496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767002.168400:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767002.168414:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767002/real 1775767002] req@ffff8b23864fe680 x1862022099002496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767002.168429:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099002496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767002.168440:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767002.168444:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767002.168448:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099002496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767003.063642:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767004.087630:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767005.111696:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767006.135670:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767007.159651:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767007.287581:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767007.287591:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767007.287595:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767007.287607:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4468 00000100:00080000:2.0:1775767007.287617:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767007.287640:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767007.287681:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099002624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767007.287727:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767007.287730:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767007.287731:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767007.287735:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4468 00000100:00080000:2.0:1775767007.287738:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767007.287740:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767007.287749:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767007.287758:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099002752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767007.287768:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767007.287771:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767007.287772:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767007.287775:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4468 00000100:00080000:2.0:1775767007.287778:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767007.287783:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767007.287784:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099002624:192.168.202.128@tcp:250:kworker.0 00000008:00100000:2.0:1775767007.287787:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767007.287798:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099002880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767007.287810:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767007.287814:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767007.287815:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767007.287818:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4468 00000100:00080000:2.0:1775767007.287821:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767007.287824:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767007.287827:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767007.287835:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099003008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767007.287842:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767007.287845:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767007.287849:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775767007.287850:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775767007.287855:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4468 00000100:00080000:2.0:1775767007.287859:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767007.287862:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767007.287865:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767007.287873:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099003136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767007.287896:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767007/real 1775767007] req@ffff8b23b9ea4a80 x1862022099002624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767007.287918:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099002624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767007.287926:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767007.287930:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767007.287933:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099002624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767007.287940:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099002752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767007.287957:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767007.287966:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767007/real 1775767007] req@ffff8b23b9ea5880 x1862022099002752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767007.287976:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099002752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767007.287982:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767007.287984:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767007.287987:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099002752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767007.288005:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099002880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767007.288017:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767007.288023:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767007/real 1775767007] req@ffff8b23b9ea4000 x1862022099002880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767007.288032:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099002880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767007.288043:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767007.288046:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767007.288050:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099002880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767007.288057:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099003008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767007.288080:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767007.288092:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767007/real 1775767007] req@ffff8b23b9ea6a00 x1862022099003008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767007.288118:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099003008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767007.288127:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767007.288130:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767007.288133:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099003008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767007.288138:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099003136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767007.288156:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767007.288164:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767007/real 1775767007] req@ffff8b23b9ea4700 x1862022099003136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767007.288172:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099003136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767007.288178:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767007.288180:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767007.288183:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099003136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767008.183736:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767009.207593:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767010.231656:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767011.255674:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767012.279633:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767012.407586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767012.407595:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767012.407598:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767012.407607:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4473 00000100:00080000:1.0:1775767012.407617:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767012.407624:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767012.407661:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099003264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767012.407685:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767012.407689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767012.407691:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767012.407695:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4473 00000100:00080000:1.0:1775767012.407699:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767012.407704:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767012.407714:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767012.407730:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099003264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775767012.407731:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099003392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767012.407768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767012.407772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767012.407775:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767012.407781:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4473 00000100:00080000:1.0:1775767012.407799:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775767012.407822:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775767012.407832:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767012.407840:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767012.407857:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099003520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767012.407867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767012.407869:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767012/real 1775767012] req@ffff8b23864fdc00 x1862022099003264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767012.407875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767012.407878:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767012.407888:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4473 00000100:00080000:1.0:1775767012.407894:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767012.407898:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767012.407903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767012.407906:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099003264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767012.407915:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099003648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767012.407927:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767012.407929:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767012.407934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767012.407938:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00080000:1.0:1775767012.407938:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767012.407947:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4473 00000100:00100000:0.0:1775767012.407949:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099003264:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767012.407950:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767012.407957:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767012.407960:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767012.407976:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099003392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767012.407996:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767012.408006:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767012/real 1775767012] req@ffff8b23864fd500 x1862022099003392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767012.408015:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099003392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767012.408021:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767012.408025:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:1.0:1775767012.408029:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099003776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767012.408030:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099003392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767012.408040:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099003520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767012.408056:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767012.408064:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767012/real 1775767012] req@ffff8b23864ff100 x1862022099003520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767012.408073:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099003520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767012.408079:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767012.408081:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767012.408083:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099003520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767012.408087:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099003648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767012.408098:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767012.408103:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767012/real 1775767012] req@ffff8b23864fe300 x1862022099003648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767012.408111:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099003648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767012.408116:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767012.408117:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767012.408119:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099003648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767012.408132:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099003776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767012.408142:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767012.408147:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767012/real 1775767012] req@ffff8b23864ffb80 x1862022099003776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767012.408160:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099003776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767012.408165:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767012.408166:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767012.408168:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099003776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767013.303645:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767014.327629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767015.351632:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767016.375635:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767017.399666:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767017.527569:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767017.527597:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767017.527602:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767017.527613:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4478 00000100:00080000:3.0:1775767017.527627:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767017.527634:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767017.527686:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099003904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767017.527728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767017.527731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767017.527732:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767017.527736:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4478 00000100:00080000:3.0:1775767017.527739:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767017.527743:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767017.527760:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767017.527770:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099003904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767017.527778:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099004032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767017.527788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767017.527792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767017.527793:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767017.527797:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4478 00000100:00080000:3.0:1775767017.527800:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767017.527803:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767017.527807:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767017.527817:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099004160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767017.527824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767017.527827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767017.527828:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767017.527831:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4478 00000100:00080000:3.0:1775767017.527834:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767017.527837:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767017.527840:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767017.527848:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767017.527850:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099004288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767017.527859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767017.527862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767017.527864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767017.527869:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4478 00000100:00080000:3.0:1775767017.527872:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767017.527875:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767017.527879:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767017.527886:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099004416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767017.527898:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767017/real 1775767017] req@ffff8b23864f2d80 x1862022099003904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767017.527917:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099003904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767017.527927:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767017.527930:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767017.527933:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099003904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767017.527959:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099004032:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767017.527975:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767017.527984:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767017/real 1775767017] req@ffff8b23864f3100 x1862022099004032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767017.527998:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099004032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767017.528004:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767017.528006:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767017.528008:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099004032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767017.528012:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099004160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767017.528023:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767017.528029:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767017/real 1775767017] req@ffff8b23864f0700 x1862022099004160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767017.528037:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099004160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767017.528042:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767017.528044:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767017.528046:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099004160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767017.528050:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099004288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767017.528079:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767017.528086:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767017/real 1775767017] req@ffff8b23864f2680 x1862022099004288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767017.528094:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099004288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767017.528106:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767017.528108:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767017.528111:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099004288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767017.528115:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099004416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767017.528127:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767017.528132:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767017/real 1775767017] req@ffff8b23864f0380 x1862022099004416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767017.528141:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099004416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767017.528147:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767017.528149:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767017.528151:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099004416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767018.423611:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767019.447686:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767020.471676:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767021.495638:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767022.519686:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767022.647585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767022.647596:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767022.647601:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767022.647612:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4484 00000100:00080000:3.0:1775767022.647625:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767022.647632:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767022.647676:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099004544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767022.647706:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767022.647709:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767022.647710:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767022.647713:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4484 00000100:00080000:3.0:1775767022.647715:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767022.647717:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767022.647726:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767022.647737:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099004672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767022.647752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767022.647760:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099004544:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767022.647763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767022.647767:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767022.647776:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4484 00000100:00080000:3.0:1775767022.647779:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767022.647782:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767022.647785:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767022.647796:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099004800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767022.647802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767022.647805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767022.647807:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767022.647810:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4484 00000100:00080000:3.0:1775767022.647811:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767022.647814:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767022.647818:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767022.647825:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099004928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767022.647831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767022.647834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767022.647835:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767022.647837:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4484 00000100:00080000:3.0:1775767022.647841:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767022.647843:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767022.647846:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767022.647855:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767022.647856:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099005056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767022.647910:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767022/real 1775767022] req@ffff8b23864f1c00 x1862022099004544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767022.647934:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099004544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767022.647946:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767022.647952:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767022.647957:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099004544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767022.647987:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099004672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767022.648013:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767022.648025:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767022/real 1775767022] req@ffff8b23864f1f80 x1862022099004672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767022.648034:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099004672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767022.648040:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767022.648043:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767022.648054:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099004672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767022.648059:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099004800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767022.648074:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767022.648081:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767022/real 1775767022] req@ffff8b23864f0a80 x1862022099004800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767022.648089:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099004800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767022.648095:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767022.648097:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767022.648099:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099004800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767022.648103:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099004928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767022.648115:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767022.648121:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767022/real 1775767022] req@ffff8b23864f0380 x1862022099004928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767022.648129:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099004928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767022.648134:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767022.648136:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767022.648137:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099004928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767022.648141:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099005056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767022.648151:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767022.648156:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767022/real 1775767022] req@ffff8b23864f2680 x1862022099005056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767022.648163:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099005056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767022.648168:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767022.648170:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767022.648172:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099005056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767023.543649:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767024.567688:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767025.591684:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767026.615628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767027.639593:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767027.767618:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767027.767627:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767027.767632:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767027.767643:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4489 00000100:00080000:0.0:1775767027.767655:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767027.767669:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767027.767713:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099005184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767027.767746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767027.767750:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767027.767753:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767027.767759:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4489 00000100:00080000:0.0:1775767027.767763:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767027.767768:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767027.767785:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767027.767792:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099005184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767027.767803:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099005312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767027.767818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767027.767822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767027.767824:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767027.767831:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4489 00000100:00080000:0.0:1775767027.767834:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767027.767839:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767027.767844:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767027.767857:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099005440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767027.767880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767027.767883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767027.767885:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775767027.767891:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775767027.767893:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4489 00000100:00080000:0.0:1775767027.767900:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767027.767904:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767027.767912:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767027.767926:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099005568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767027.767937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767027.767941:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767027.767944:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767027.767948:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767027/real 1775767027] req@ffff8b2382daed80 x1862022099005184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767027.767952:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4489 00000100:00080000:0.0:1775767027.767968:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767027.767976:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767027.767979:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099005184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775767027.767984:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775767027.767989:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767027.767996:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767027.768000:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099005184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767027.768008:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099005696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767027.768032:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099005312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767027.768053:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767027.768062:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767027/real 1775767027] req@ffff8b2382daea00 x1862022099005312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767027.768070:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099005312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767027.768076:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767027.768078:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767027.768081:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099005312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767027.768086:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099005440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767027.768099:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767027.768105:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767027/real 1775767027] req@ffff8b2382dad500 x1862022099005440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767027.768112:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099005440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767027.768117:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767027.768119:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767027.768121:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099005440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767027.768125:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099005568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767027.768137:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767027.768142:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767027/real 1775767027] req@ffff8b2382dae680 x1862022099005568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767027.768150:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099005568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767027.768155:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767027.768157:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767027.768158:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099005568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767027.768162:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099005696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767027.768172:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767027.768178:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767027/real 1775767027] req@ffff8b2382dac380 x1862022099005696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767027.768186:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099005696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767027.768191:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767027.768193:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767027.768194:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099005696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767028.663690:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767029.687667:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767030.711723:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767031.735708:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767032.759665:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767032.887630:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767032.887640:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767032.887645:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767032.887657:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4494 00000100:00080000:3.0:1775767032.887670:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767032.887677:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767032.887740:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099005824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767032.887776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767032.887781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767032.887783:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767032.887787:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4494 00000100:00080000:3.0:1775767032.887792:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767032.887797:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767032.887810:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767032.887822:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099005824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767032.887828:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099005952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767032.887842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767032.887844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767032.887846:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767032.887850:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4494 00000100:00080000:3.0:1775767032.887863:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767032.887866:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767032.887869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767032.887880:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099006080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767032.887887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767032.887890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767032.887891:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767032.887894:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4494 00000100:00080000:3.0:1775767032.887896:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767032.887898:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767032.887902:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767032.887909:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099006208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767032.887916:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767032.887921:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767032.887924:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767032.887925:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767032.887928:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4494 00000100:00080000:3.0:1775767032.887932:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767032.887935:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767032.887939:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767032.887947:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099006336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767032.887968:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767032/real 1775767032] req@ffff8b23864f0700 x1862022099005824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767032.887984:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099005824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767032.887994:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767032.887997:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767032.888001:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099005824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767032.888022:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099005952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767032.888038:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767032.888046:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767032/real 1775767032] req@ffff8b23864f3100 x1862022099005952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767032.888055:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099005952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767032.888060:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767032.888063:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767032.888065:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099005952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767032.888068:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099006080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767032.888080:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767032.888085:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767032/real 1775767032] req@ffff8b23864f2d80 x1862022099006080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767032.888104:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099006080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767032.888114:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767032.888116:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767032.888118:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099006080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767032.888122:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099006208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767032.888136:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767032.888141:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767032/real 1775767032] req@ffff8b23864f3b80 x1862022099006208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767032.888148:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099006208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767032.888154:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767032.888156:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767032.888157:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099006208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767032.888176:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099006336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767032.888200:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767032.888207:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767032/real 1775767032] req@ffff8b23864f1180 x1862022099006336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767032.888215:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099006336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767032.888220:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767032.888222:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767032.888225:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099006336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767033.783625:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767034.807682:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767035.831625:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767036.855761:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767037.879599:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767038.007616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767038.007628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767038.007632:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767038.007643:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4499 00000100:00080000:0.0:1775767038.007655:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767038.007661:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767038.007697:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022099006464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767038.007722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767038.007725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767038.007726:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767038.007729:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4499 00000100:00080000:0.0:1775767038.007731:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767038.007733:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767038.007741:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767038.007750:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099006592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767038.007758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767038.007762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767038.007764:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767038.007766:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099006464:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767038.007770:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4499 00000100:00080000:0.0:1775767038.007774:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767038.007778:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767038.007780:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767038.007790:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022099006720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767038.007796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767038.007799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767038.007800:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767038.007803:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4499 00000100:00080000:0.0:1775767038.007805:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767038.007807:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767038.007810:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767038.007816:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022099006848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767038.007822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767038.007825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767038.007826:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767038.007829:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4499 00000100:00080000:0.0:1775767038.007831:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767038.007833:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767038.007836:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767038.007840:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767038.007846:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099006976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767038.007885:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767038/real 1775767038] req@ffff8b2382dad880 x1862022099006464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767038.007907:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022099006464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767038.007918:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767038.007922:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767038.007926:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099006464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767038.007950:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099006592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767038.007973:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767038.007985:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767038/real 1775767038] req@ffff8b2382daca80 x1862022099006592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767038.007998:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099006592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767038.008007:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767038.008010:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767038.008014:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099006592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767038.008020:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099006720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767038.008041:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767038.008052:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767038/real 1775767038] req@ffff8b2382daf100 x1862022099006720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767038.008064:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022099006720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767038.008072:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767038.008075:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767038.008078:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099006720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767038.008084:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099006848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767038.008105:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767038.008129:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767038/real 1775767038] req@ffff8b2382dac000 x1862022099006848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767038.008142:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022099006848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767038.008149:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767038.008152:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767038.008155:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099006848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767038.008162:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099006976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767038.008182:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767038.008193:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767038/real 1775767038] req@ffff8b2382dac380 x1862022099006976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767038.008205:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099006976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767038.008213:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767038.008216:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767038.008219:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099006976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767038.903710:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767039.927614:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767040.951652:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767041.975622:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767042.999638:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767043.127516:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767043.127526:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767043.127531:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767043.127542:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4504 00000100:00080000:0.0:1775767043.127555:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767043.127562:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767043.127604:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099007104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767043.127643:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767043.127646:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767043.127648:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767043.127651:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4504 00000100:00080000:0.0:1775767043.127653:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767043.127656:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767043.127665:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767043.127675:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099007232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767043.127684:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767043.127686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767043.127688:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767043.127691:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4504 00000100:00080000:0.0:1775767043.127694:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767043.127697:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767043.127701:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767043.127713:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099007360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767043.127720:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099007104:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767043.127722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767043.127727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767043.127729:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767043.127732:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4504 00000100:00080000:0.0:1775767043.127734:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767043.127736:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767043.127739:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767043.127747:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099007488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767043.127753:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767043.127756:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767043.127757:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767043.127759:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4504 00000100:00080000:0.0:1775767043.127762:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767043.127764:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767043.127767:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767043.127774:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099007616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767043.127798:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767043.127850:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767043/real 1775767043] req@ffff8b2382dae680 x1862022099007104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767043.127867:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099007104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767043.127876:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767043.127879:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767043.127883:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099007104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767043.127889:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099007232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767043.127904:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767043.127912:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767043/real 1775767043] req@ffff8b2382dad500 x1862022099007232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767043.127920:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099007232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767043.127930:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767043.127933:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767043.127935:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099007232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767043.127959:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099007360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767043.127970:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767043.127975:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767043/real 1775767043] req@ffff8b2382daea00 x1862022099007360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767043.127982:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099007360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767043.127988:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767043.127990:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767043.127992:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099007360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767043.127996:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099007488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767043.128007:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767043.128012:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767043/real 1775767043] req@ffff8b2382daed80 x1862022099007488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767043.128020:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099007488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767043.128024:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767043.128026:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767043.128029:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099007488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767043.128032:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099007616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767043.128041:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767043.128046:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767043/real 1775767043] req@ffff8b2382daf480 x1862022099007616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767043.128052:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099007616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767043.128073:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767043.128075:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767043.128078:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099007616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767044.023636:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767045.047718:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767046.071830:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767047.095683:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767048.119625:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767048.247558:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767048.247569:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767048.247573:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767048.247583:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4509 00000100:00080000:2.0:1775767048.247595:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767048.247602:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767048.247644:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022099007744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767048.247674:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767048.247677:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767048.247678:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767048.247687:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4509 00000100:00080000:2.0:1775767048.247690:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767048.247693:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767048.247701:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767048.247710:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099007872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767048.247719:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767048.247721:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767048.247723:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767048.247727:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4509 00000100:00080000:2.0:1775767048.247729:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767048.247732:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767048.247735:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767048.247741:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099008000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767048.247747:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099007744:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767048.247750:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767048.247754:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767048.247756:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767048.247759:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4509 00000100:00080000:2.0:1775767048.247761:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767048.247763:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767048.247766:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767048.247774:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099008128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767048.247780:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767048.247782:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767048.247784:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767048.247786:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4509 00000100:00080000:2.0:1775767048.247797:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767048.247799:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767048.247803:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767048.247812:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099008256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767048.247855:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099007872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767048.247895:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099008000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767048.247922:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099008128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767048.247936:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099008256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767049.143716:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767050.167627:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767051.191665:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775767051.319715:0:2364:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:0.0:1775767051.319762:0:2364:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775767051.319780:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:0.0:1775767051.319821:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:0.0:1775767051.319835:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:0.0:1775767051.319843:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:0.0:1775767051.319853:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:3.0:1775767051.319862:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767048/real 1775767051] req@ffff8b23b9ea6d80 x1862022099008000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767051.319893:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099008000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767051.319912:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767051.319919:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767051.319923:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099008000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767051.319932:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767048/real 1775767051] req@ffff8b23b9ea6680 x1862022099008128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767051.319946:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099008128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767051.319954:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767051.319958:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767051.319962:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099008128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767051.319984:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767048/real 1775767051] req@ffff8b23b9ea5f80 x1862022099008256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767051.319999:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099008256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767051.320008:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767051.320013:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767051.320016:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099008256:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767051.320024:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767048/real 1775767051] req@ffff8b23b9ea5500 x1862022099007744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767051.320036:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022099007744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767051.320044:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767051.320048:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767051.320051:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099007744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767051.320057:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767048/real 1775767051] req@ffff8b23b9ea5c00 x1862022099007872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767051.320069:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099007872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767051.320077:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767051.320080:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767051.320084:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099007872:192.168.202.128@tcp:38:kworker.0 00010000:00010000:0.0:1775767052.215652:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767053.239700:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767053.367559:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767053.367567:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767053.367571:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767053.367580:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4514 00000100:00080000:3.0:1775767053.367591:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767053.367597:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767053.367632:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099008384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767053.367657:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767053.367660:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767053.367661:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767053.367664:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4514 00000100:00080000:3.0:1775767053.367668:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767053.367671:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767053.367680:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767053.367689:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099008512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767053.367699:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767053.367702:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767053.367704:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767053.367713:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4514 00000100:00100000:0.0:1775767053.367714:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099008384:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767053.367716:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767053.367728:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767053.367733:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767053.367748:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099008640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767053.367758:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767053.367762:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767053.367764:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767053.367769:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4514 00000100:00080000:3.0:1775767053.367774:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767053.367779:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767053.367785:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767053.367804:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767053.367806:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099008768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767053.367828:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767053.367834:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767053.367837:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767053.367844:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4514 00000100:00080000:3.0:1775767053.367851:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767053.367854:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767053/real 1775767053] req@ffff8b23864f3480 x1862022099008384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767053.367864:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767053.367880:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767053.367890:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099008384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767053.367900:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099008896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767053.367961:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767053.367972:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767053.367980:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099008384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767053.368023:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099008512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767053.368057:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767053.368075:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767053/real 1775767053] req@ffff8b23864f2a00 x1862022099008512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767053.368088:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099008512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767053.368096:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767053.368100:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767053.368104:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099008512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767053.368110:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099008640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767053.368137:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767053.368148:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767053/real 1775767053] req@ffff8b23864f0000 x1862022099008640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767053.368162:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099008640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767053.368170:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767053.368175:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767053.368178:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099008640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767053.368185:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099008768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767053.368215:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767053.368227:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767053/real 1775767053] req@ffff8b23864f1180 x1862022099008768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767053.368238:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099008768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767053.368247:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767053.368251:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767053.368267:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099008768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767053.368274:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099008896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767053.368300:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767053.368310:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767053/real 1775767053] req@ffff8b23864f3b80 x1862022099008896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767053.368322:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099008896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767053.368329:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767053.368333:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767053.368336:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099008896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767054.263605:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767055.287643:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767056.311711:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767057.335654:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767058.359747:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767058.487549:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767058.487559:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767058.487563:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767058.487574:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4519 00000100:00080000:3.0:1775767058.487587:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767058.487595:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767058.487635:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099009024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767058.487666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767058.487670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767058.487672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767058.487676:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4519 00000100:00080000:3.0:1775767058.487682:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767058.487688:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767058.487701:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767058.487720:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099009024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767058.487720:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099009152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767058.487745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767058.487749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767058.487751:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767058.487757:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4519 00000100:00080000:3.0:1775767058.487762:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767058.487767:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767058.487773:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767058.487799:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099009280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767058.487807:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767058.487823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767058.487827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767058.487830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767058.487837:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4519 00000100:00080000:3.0:1775767058.487842:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767058.487847:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767058.487862:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767058.487865:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767058/real 1775767058] req@ffff8b23864f2d80 x1862022099009024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767058.487884:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099009408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767058.487890:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099009024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767058.487918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767058.487923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767058.487927:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767058.487941:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4519 00000100:00080000:3.0:1775767058.487960:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767058.487962:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767058.487971:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00080000:3.0:1775767058.487972:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767058.487980:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099009024:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775767058.487985:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767058.488005:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099009536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767058.488012:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099009152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767058.488049:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767058.488062:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767058/real 1775767058] req@ffff8b23864f3100 x1862022099009152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767058.488072:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099009152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767058.488078:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767058.488081:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767058.488083:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099009152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767058.488088:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099009280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767058.488100:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767058.488106:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767058/real 1775767058] req@ffff8b23864f0700 x1862022099009280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767058.488114:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099009280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767058.488119:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767058.488121:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767058.488123:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099009280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767058.488128:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099009408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767058.488141:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767058.488147:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767058/real 1775767058] req@ffff8b23864f2680 x1862022099009408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767058.488154:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099009408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767058.488158:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767058.488160:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767058.488162:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099009408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767058.488173:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099009536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767058.488183:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767058.488189:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767058/real 1775767058] req@ffff8b23864f0380 x1862022099009536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767058.488196:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099009536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767058.488201:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767058.488203:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767058.488204:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099009536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767059.383646:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767060.407637:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767061.431712:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767062.455715:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767063.479628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767063.607626:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767063.607638:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767063.607644:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767063.607654:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4525 00000100:00080000:0.0:1775767063.607667:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767063.607673:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767063.607719:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022099009664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767063.607749:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767063.607752:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767063.607753:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767063.607756:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4525 00000100:00080000:0.0:1775767063.607759:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767063.607762:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767063.607772:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767063.607783:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022099009792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767063.607789:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099009664:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767063.607795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767063.607799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767063.607800:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767063.607805:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4525 00000100:00080000:0.0:1775767063.607807:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767063.607809:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767063.607813:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767063.607823:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022099009920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767063.607831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767063.607834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767063.607835:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767063.607838:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4525 00000100:00080000:0.0:1775767063.607854:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767063.607857:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767063.607860:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767063.607868:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022099010048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767063.607874:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767063.607877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767063.607878:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767063.607880:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4525 00000100:00080000:0.0:1775767063.607886:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775767063.607886:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775767063.607892:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767063.607895:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767063.607905:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099010176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767063.607937:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767063/real 1775767063] req@ffff8b2382dafb80 x1862022099009664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767063.607954:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022099009664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767063.607962:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767063.607965:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767063.607968:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099009664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767063.607991:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099009792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767063.608009:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767063.608021:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767063/real 1775767063] req@ffff8b2382dadf80 x1862022099009792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767063.608034:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022099009792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767063.608050:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767063.608054:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767063.608058:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099009792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767063.608065:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099009920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767063.608086:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767063.608094:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767063/real 1775767063] req@ffff8b2382dace00 x1862022099009920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767063.608102:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022099009920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767063.608108:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767063.608111:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767063.608113:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099009920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767063.608117:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099010048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767063.608131:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767063.608136:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767063/real 1775767063] req@ffff8b2382dac700 x1862022099010048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767063.608144:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022099010048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767063.608148:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767063.608150:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767063.608153:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099010048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767063.608157:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099010176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767063.608167:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767063.608172:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767063/real 1775767063] req@ffff8b2382daf480 x1862022099010176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767063.608179:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099010176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767063.608184:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767063.608186:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767063.608188:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099010176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767064.503669:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767065.527640:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767066.551685:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767067.575625:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767068.599654:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767068.727614:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767068.727625:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767068.727630:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767068.727640:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4530 00000100:00080000:3.0:1775767068.727653:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767068.727660:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767068.727705:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099010304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767068.727735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767068.727738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767068.727739:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767068.727742:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4530 00000100:00080000:3.0:1775767068.727745:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767068.727749:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767068.727760:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767068.727768:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099010432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767068.727774:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099010304:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767068.727781:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767068.727798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767068.727800:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767068.727806:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4530 00000100:00080000:3.0:1775767068.727811:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767068.727816:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767068.727822:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767068.727837:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099010560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767068.727849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767068.727853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767068.727855:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767068.727860:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4530 00000800:00000100:0.0:1775767068.727864:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767068.727867:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767068.727873:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767068.727879:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767068.727890:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099010688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767068.727897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767068.727900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767068.727902:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767068.727905:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4530 00000100:00080000:3.0:1775767068.727908:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767068.727913:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767068/real 1775767068] req@ffff8b23864f0a80 x1862022099010304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767068.727913:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767068.727919:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767068.727932:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099010816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767068.727937:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099010304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767068.727951:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767068.727955:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767068.727959:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099010304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767068.727980:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099010432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767068.727995:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767068.728003:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767068/real 1775767068] req@ffff8b23864f1f80 x1862022099010432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767068.728012:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099010432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767068.728017:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767068.728020:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767068.728022:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099010432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767068.728025:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099010560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767068.728036:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767068.728042:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767068/real 1775767068] req@ffff8b23864f1c00 x1862022099010560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767068.728054:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099010560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767068.728059:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767068.728061:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767068.728063:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099010560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767068.728066:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099010688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767068.728078:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767068.728083:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767068/real 1775767068] req@ffff8b23864f0380 x1862022099010688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767068.728090:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099010688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767068.728095:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767068.728096:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767068.728098:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099010688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767068.728101:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099010816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767068.728114:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767068.728123:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767068/real 1775767068] req@ffff8b23864f2680 x1862022099010816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767068.728134:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099010816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767068.728147:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767068.728150:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767068.728152:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099010816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767069.623644:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767070.647641:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767071.671629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767072.695594:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767073.719787:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767073.847592:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767073.847607:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767073.847614:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767073.847629:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4535 00000100:00080000:3.0:1775767073.847647:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767073.847657:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767073.847723:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099010944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767073.847795:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767073.847801:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767073.847803:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767073.847809:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4535 00000100:00080000:3.0:1775767073.847813:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767073.847819:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767073.847833:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767073.847851:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099011072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767073.847869:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767073.847874:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767073.847877:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767073.847883:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4535 00000100:00080000:3.0:1775767073.847887:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767073.847892:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767073.847898:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767073.847905:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099010944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767073.847917:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099011200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767073.847931:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767073.847935:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767073.847937:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767073.847942:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4535 00000100:00080000:3.0:1775767073.847946:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767073.847949:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767073.847955:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767073.847968:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099011328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767073.847979:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767073.847982:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767073.847984:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767073.847988:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4535 00000100:00080000:3.0:1775767073.847994:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767073.847998:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767073.848005:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767073.848016:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099011456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767073.848051:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767073.848112:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767073/real 1775767073] req@ffff8b23864f0700 x1862022099010944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767073.848139:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099010944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767073.848153:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767073.848159:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767073.848164:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099010944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767073.848174:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099011072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767073.848201:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767073.848215:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767073/real 1775767073] req@ffff8b23864f3100 x1862022099011072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767073.848230:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099011072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767073.848239:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767073.848245:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767073.848249:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099011072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767073.848280:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099011200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767073.848300:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767073.848312:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767073/real 1775767073] req@ffff8b23864f2d80 x1862022099011200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767073.848326:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099011200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767073.848335:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767073.848339:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767073.848343:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099011200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767073.848349:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099011328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767073.848369:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767073.848379:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767073/real 1775767073] req@ffff8b23864f3b80 x1862022099011328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767073.848392:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099011328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767073.848402:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767073.848405:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767073.848409:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099011328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767073.848415:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099011456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767073.848433:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767073.848443:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767073/real 1775767073] req@ffff8b23864f1180 x1862022099011456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767073.848508:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099011456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767073.848519:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767073.848523:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767073.848527:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099011456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767074.743677:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767075.767690:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767076.791648:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767077.815809:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767078.839700:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767078.967652:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767078.967668:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767078.967675:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767078.967737:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4540 00000100:00080000:1.0:1775767078.967752:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767078.967760:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767078.967811:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099011584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767078.967846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767078.967851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767078.967853:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767078.967858:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4540 00000100:00080000:1.0:1775767078.967863:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767078.967868:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767078.967879:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767078.967887:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099011584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775767078.967899:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099011712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767078.967914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767078.967918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767078.967920:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767078.967925:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4540 00000100:00080000:1.0:1775767078.967948:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767078.967954:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775767078.967959:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:1.0:1775767078.967960:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767078.967979:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099011840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767078.967991:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767078.967994:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767078.967996:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767078.968002:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4540 00000100:00100000:0.0:1775767078.968004:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767078/real 1775767078] req@ffff8b23864ffb80 x1862022099011584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767078.968010:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767078.968018:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767078.968024:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099011584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775767078.968025:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775767078.968038:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767078.968041:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:1.0:1775767078.968046:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099011968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767078.968047:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099011584:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767078.968060:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767078.968065:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767078.968067:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767078.968073:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4540 00000100:00080000:1.0:1775767078.968077:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767078.968082:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767078.968085:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099011712:192.168.202.128@tcp:38:kworker.0 00000008:00100000:1.0:1775767078.968090:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767078.968108:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775767078.968109:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099012096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767078.968121:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767078/real 1775767078] req@ffff8b23864fe300 x1862022099011712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767078.968137:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099011712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767078.968146:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767078.968150:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767078.968155:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099011712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767078.968162:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099011840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767078.968185:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767078.968196:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767078/real 1775767078] req@ffff8b23864ff100 x1862022099011840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767078.968208:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099011840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767078.968214:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767078.968217:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767078.968219:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099011840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767078.968223:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099011968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767078.968239:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767078.968245:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767078/real 1775767078] req@ffff8b23864fd500 x1862022099011968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767078.968253:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099011968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767078.968259:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767078.968261:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767078.968263:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099011968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767078.968274:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099012096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767078.968285:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767078.968290:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767078/real 1775767078] req@ffff8b23864fdc00 x1862022099012096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767078.968297:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099012096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767078.968302:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767078.968304:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767078.968306:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099012096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767079.863653:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767080.887657:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767081.911727:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767082.935624:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767083.959696:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767084.087664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767084.087672:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767084.087676:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767084.087686:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4545 00000100:00080000:3.0:1775767084.087697:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767084.087703:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767084.087741:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099012224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767084.087766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767084.087770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767084.087771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767084.087774:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4545 00000100:00080000:3.0:1775767084.087777:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767084.087779:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767084.087787:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767084.087797:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099012352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767084.087807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767084.087813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767084.087816:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767084.087819:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099012224:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767084.087823:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4545 00000100:00080000:3.0:1775767084.087829:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767084.087831:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767084.087835:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767084.087844:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099012480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767084.087852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767084.087856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767084.087857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767084.087861:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4545 00000100:00080000:3.0:1775767084.087864:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767084.087866:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767084.087869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767084.087879:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099012608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767084.087885:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767084.087892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767084.087895:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767084.087897:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767084.087901:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4545 00000100:00080000:3.0:1775767084.087905:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767084.087908:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767084.087912:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767084.087920:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099012736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767084.087930:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767084/real 1775767084] req@ffff8b23864f0000 x1862022099012224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767084.087949:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099012224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767084.087957:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767084.087961:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767084.087965:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099012224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767084.087971:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099012352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767084.087986:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767084.087995:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767084/real 1775767084] req@ffff8b23864f2a00 x1862022099012352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767084.088004:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099012352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767084.088009:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767084.088011:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767084.088014:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099012352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767084.088032:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099012480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767084.088044:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767084.088051:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767084/real 1775767084] req@ffff8b23864f3480 x1862022099012480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767084.088075:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099012480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767084.088081:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767084.088083:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767084.088085:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099012480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767084.088090:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099012608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767084.088104:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767084.088110:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767084/real 1775767084] req@ffff8b23864f1180 x1862022099012608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767084.088118:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099012608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767084.088124:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767084.088126:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767084.088128:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099012608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767084.088132:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099012736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767084.088143:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767084.088148:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767084/real 1775767084] req@ffff8b23864f3b80 x1862022099012736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767084.088156:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099012736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767084.088161:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767084.088163:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767084.088165:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099012736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767084.983693:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767086.007681:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767087.031748:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767088.055642:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767089.079689:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767089.207614:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767089.207628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767089.207635:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767089.207649:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4550 00000100:00080000:3.0:1775767089.207665:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767089.207674:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767089.207723:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099012864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767089.207754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767089.207757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767089.207759:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767089.207762:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4550 00000100:00080000:3.0:1775767089.207765:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767089.207768:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767089.207777:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767089.207788:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099012992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767089.207797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767089.207800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767089.207802:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767089.207805:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4550 00000100:00080000:3.0:1775767089.207809:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767089.207812:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767089.207816:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767089.207821:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099012864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767089.207827:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099013120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767089.207837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767089.207839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767089.207840:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767089.207842:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4550 00000100:00080000:3.0:1775767089.207844:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767089.207846:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767089.207849:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767089.207857:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099013248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767089.207863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767089.207865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767089.207866:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767089.207868:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4550 00000100:00080000:3.0:1775767089.207870:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767089.207872:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767089.207876:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767089.207883:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099013376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767089.207913:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767089.207964:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767089/real 1775767089] req@ffff8b23864f2d80 x1862022099012864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767089.207986:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099012864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767089.207998:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767089.208003:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767089.208008:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099012864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767089.208015:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099012992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767089.208036:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767089.208047:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767089/real 1775767089] req@ffff8b23864f3100 x1862022099012992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767089.208059:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099012992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767089.208067:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767089.208070:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767089.208073:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099012992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767089.208099:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099013120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767089.208115:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767089.208125:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767089/real 1775767089] req@ffff8b23864f0700 x1862022099013120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767089.208136:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099013120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767089.208150:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767089.208153:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767089.208156:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099013120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767089.208161:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099013248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767089.208176:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767089.208185:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767089/real 1775767089] req@ffff8b23864f2680 x1862022099013248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767089.208197:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099013248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767089.208205:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767089.208208:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767089.208212:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099013248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767089.208217:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099013376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767089.208233:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767089.208242:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767089/real 1775767089] req@ffff8b23864f0380 x1862022099013376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767089.208254:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099013376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767089.208262:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767089.208265:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767089.208269:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099013376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767090.103700:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767091.127693:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767092.151687:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767093.175719:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767094.199686:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767094.327667:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767094.327678:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767094.327684:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767094.327695:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4555 00000100:00080000:1.0:1775767094.327709:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767094.327716:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767094.327768:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099013504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767094.327797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767094.327800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767094.327802:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767094.327804:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4555 00000100:00080000:1.0:1775767094.327807:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767094.327810:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767094.327819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767094.327834:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099013632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767094.327844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767094.327846:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767094.327848:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767094.327851:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4555 00000100:00080000:1.0:1775767094.327853:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767094.327856:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767094.327860:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767094.327868:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099013760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767094.327877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767094.327882:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767094.327885:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767094.327886:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099013504:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767094.327890:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4555 00000100:00080000:1.0:1775767094.327897:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767094.327907:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767094.327910:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767094.327922:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099013888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767094.327928:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767094.327931:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767094.327932:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767094.327935:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4555 00000100:00080000:1.0:1775767094.327940:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767094.327943:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767094.327946:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767094.327953:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099014016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767094.327977:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767094.328036:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767094/real 1775767094] req@ffff8b23864fe680 x1862022099013504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767094.328051:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099013504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767094.328060:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767094.328063:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767094.328066:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099013504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767094.328073:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099013632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767094.328090:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767094.328099:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767094/real 1775767094] req@ffff8b23864fca80 x1862022099013632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767094.328114:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099013632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767094.328127:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767094.328130:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767094.328132:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099013632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767094.328153:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099013760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767094.328167:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767094.328173:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767094/real 1775767094] req@ffff8b23864fea00 x1862022099013760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767094.328181:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099013760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767094.328187:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767094.328189:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767094.328191:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099013760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767094.328195:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099013888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767094.328206:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767094.328212:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767094/real 1775767094] req@ffff8b23864fd180 x1862022099013888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767094.328219:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099013888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767094.328224:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767094.328226:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767094.328227:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099013888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767094.328231:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099014016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767094.328241:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767094.328246:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767094/real 1775767094] req@ffff8b23864fce00 x1862022099014016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767094.328258:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099014016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767094.328263:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767094.328265:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767094.328267:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099014016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767095.223693:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767096.247658:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767097.271677:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767098.295662:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767099.319687:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767099.447581:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767099.447589:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767099.447593:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767099.447602:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4560 00000100:00080000:1.0:1775767099.447612:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767099.447619:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767099.447663:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099014144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767099.447688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767099.447691:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767099.447692:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767099.447695:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4560 00000100:00080000:1.0:1775767099.447697:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767099.447699:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767099.447706:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767099.447720:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099014272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767099.447730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767099.447733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767099.447735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767099.447738:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099014144:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767099.447741:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4560 00000100:00080000:1.0:1775767099.447746:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767099.447749:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767099.447753:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767099.447769:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099014400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767099.447779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767099.447782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767099.447785:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775767099.447797:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775767099.447803:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4560 00000100:00080000:1.0:1775767099.447810:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767099.447816:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767099.447822:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767099.447835:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099014528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767099.447845:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767099/real 1775767099] req@ffff8b23864fdf80 x1862022099014144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767099.447851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767099.447858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767099.447861:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767099.447866:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4560 00000100:00080000:1.0:1775767099.447869:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767099.447872:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767099.447875:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099014144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775767099.447878:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775767099.447889:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1775767099.447894:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099014656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767099.447897:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767099.447903:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099014144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767099.447927:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099014272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767099.447947:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767099.447956:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767099/real 1775767099] req@ffff8b23864ff800 x1862022099014272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767099.447969:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099014272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767099.447975:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767099.447977:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767099.447979:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099014272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767099.447983:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099014400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767099.447993:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767099.447999:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767099/real 1775767099] req@ffff8b23864fed80 x1862022099014400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767099.448006:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099014400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767099.448011:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767099.448012:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767099.448014:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099014400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767099.448018:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099014528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767099.448036:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767099.448043:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767099/real 1775767099] req@ffff8b23864fc000 x1862022099014528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767099.448050:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099014528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767099.448060:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767099.448062:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767099.448064:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099014528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767099.448068:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099014656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767099.448086:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767099.448092:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767099/real 1775767099] req@ffff8b23864fc380 x1862022099014656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767099.448099:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099014656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767099.448104:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767099.448106:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767099.448108:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099014656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767100.343652:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767101.367728:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767102.391701:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767103.415657:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767104.439671:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767104.567617:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767104.567628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767104.567633:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767104.567644:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4565 00000100:00080000:3.0:1775767104.567657:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767104.567663:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767104.567707:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099014784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767104.567740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767104.567744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767104.567747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767104.567751:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4565 00000100:00080000:3.0:1775767104.567755:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767104.567758:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767104.567769:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767104.567785:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099014912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767104.567788:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099014784:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767104.567797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767104.567800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767104.567802:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767104.567806:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4565 00000100:00080000:3.0:1775767104.567809:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767104.567813:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767104.567818:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767104.567828:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099015040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767104.567836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767104.567839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767104.567841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767104.567845:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4565 00000100:00080000:3.0:1775767104.567848:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767104.567851:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767104.567855:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767104.567864:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767104.567865:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099015168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767104.567873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767104.567876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767104.567878:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767104.567881:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4565 00000100:00080000:3.0:1775767104.567884:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767104.567887:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767104.567891:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767104.567900:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099015296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767104.567914:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767104/real 1775767104] req@ffff8b23864f1c00 x1862022099014784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767104.567932:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099014784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767104.567943:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767104.567948:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767104.567953:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099014784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767104.567979:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099014912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767104.567998:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767104.568007:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767104/real 1775767104] req@ffff8b23864f1f80 x1862022099014912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767104.568017:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099014912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767104.568025:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767104.568028:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767104.568032:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099014912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767104.568039:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099015040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767104.568060:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767104.568072:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767104/real 1775767104] req@ffff8b23864f0a80 x1862022099015040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767104.568080:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099015040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767104.568086:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767104.568088:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767104.568090:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099015040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767104.568094:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099015168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767104.568110:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767104.568116:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767104/real 1775767104] req@ffff8b23864f0380 x1862022099015168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767104.568124:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099015168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767104.568128:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767104.568130:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767104.568132:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099015168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767104.568135:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099015296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767104.568145:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767104.568150:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767104/real 1775767104] req@ffff8b23864f2680 x1862022099015296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767104.568157:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099015296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767104.568163:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767104.568164:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767104.568166:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099015296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767105.463633:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767106.487604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767107.511667:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767108.535614:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767109.559613:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767109.687527:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767109.687538:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767109.687544:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767109.687555:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4571 00000100:00080000:3.0:1775767109.687567:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767109.687574:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767109.687617:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099015424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767109.687645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767109.687648:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767109.687649:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767109.687652:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4571 00000100:00080000:3.0:1775767109.687655:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767109.687657:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767109.687665:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767109.687675:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099015552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767109.687684:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767109.687687:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767109.687688:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767109.687692:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4571 00000100:00080000:3.0:1775767109.687694:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767109.687696:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767109.687700:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767109.687707:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099015424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767109.687709:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099015680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767109.687725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767109.687728:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767109.687728:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767109.687732:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4571 00000100:00080000:3.0:1775767109.687733:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767109.687736:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767109.687739:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767109.687747:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099015808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767109.687754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767109.687757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767109.687758:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767109.687760:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4571 00000100:00080000:3.0:1775767109.687762:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767109.687764:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767109.687766:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767109.687774:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099015936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767109.687786:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767109.687831:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767109/real 1775767109] req@ffff8b23864f0700 x1862022099015424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767109.687847:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099015424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767109.687856:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767109.687859:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767109.687862:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099015424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767109.687868:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099015552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767109.687884:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767109.687892:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767109/real 1775767109] req@ffff8b23864f3100 x1862022099015552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767109.687901:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099015552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767109.687906:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767109.687908:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767109.687909:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099015552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767109.687930:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099015680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767109.687942:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767109.687948:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767109/real 1775767109] req@ffff8b23864f2d80 x1862022099015680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767109.687956:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099015680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767109.687966:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767109.687968:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767109.687970:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099015680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767109.687974:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099015808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767109.687984:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767109.687989:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767109/real 1775767109] req@ffff8b23864f3b80 x1862022099015808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767109.687996:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099015808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767109.688002:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767109.688003:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767109.688005:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099015808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767109.688009:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099015936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767109.688018:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767109.688023:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767109/real 1775767109] req@ffff8b23864f1180 x1862022099015936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767109.688030:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099015936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767109.688034:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767109.688036:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767109.688039:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099015936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767110.583741:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767111.607636:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767112.631688:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767113.655626:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767114.679701:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767114.807576:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767114.807590:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767114.807595:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767114.807606:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4576 00000100:00080000:1.0:1775767114.807619:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767114.807626:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767114.807680:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099016064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767114.807707:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767114.807710:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767114.807712:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767114.807714:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4576 00000100:00080000:1.0:1775767114.807717:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767114.807720:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767114.807733:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767114.807743:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099016192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767114.807752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767114.807755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767114.807756:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767114.807759:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4576 00000100:00080000:1.0:1775767114.807761:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767114.807764:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767114.807767:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767114.807776:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099016320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767114.807783:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099016064:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767114.807786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767114.807791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767114.807793:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767114.807813:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4576 00000100:00080000:1.0:1775767114.807815:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767114.807817:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767114.807821:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767114.807831:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099016448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767114.807837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767114.807840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767114.807841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767114.807844:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4576 00000100:00080000:1.0:1775767114.807865:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767114.807868:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767114.807871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767114.807885:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099016576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767114.807933:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099016192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767114.807967:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099016320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767114.807986:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099016448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767114.808004:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099016576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767115.703683:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767116.727681:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767117.751687:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775767117.879882:0:2365:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775767117.879940:0:2365:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775767117.879965:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767117.880020:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767117.880031:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767117.880042:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767117.880051:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775767117.880239:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767114/real 1775767117] req@ffff8b23864fed80 x1862022099016320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767117.880304:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099016320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767117.880319:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767117.880326:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767117.880332:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099016320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767117.880345:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767114/real 1775767117] req@ffff8b23864ff800 x1862022099016448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767117.880360:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099016448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767117.880374:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767117.880379:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767117.880382:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099016448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767117.880390:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767114/real 1775767117] req@ffff8b23864fdf80 x1862022099016576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767117.880405:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099016576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767117.880413:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767117.880417:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767117.880421:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099016576:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767117.880429:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767114/real 1775767117] req@ffff8b23864fc380 x1862022099016064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767117.880443:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099016064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767117.880450:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767117.880490:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767117.880494:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099016064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767117.880501:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767114/real 1775767117] req@ffff8b23864fc000 x1862022099016192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767117.880516:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099016192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767117.880526:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767117.880530:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767117.880534:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099016192:192.168.202.128@tcp:38:kworker.0 00010000:00010000:1.0:1775767118.775675:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767119.799685:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767119.927567:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767119.927581:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767119.927588:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767119.927601:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4581 00000100:00080000:1.0:1775767119.927617:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767119.927626:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767119.927673:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099016704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767119.927707:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767119.927710:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767119.927712:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767119.927715:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4581 00000100:00080000:1.0:1775767119.927718:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767119.927721:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767119.927730:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767119.927741:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099016832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767119.927750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767119.927753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767119.927754:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767119.927758:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4581 00000100:00080000:1.0:1775767119.927759:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767119.927765:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767119.927767:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099016704:192.168.202.128@tcp:250:kworker.0 00000008:00100000:1.0:1775767119.927769:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767119.927779:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099016960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767119.927791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767119.927793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767119.927795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767119.927798:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4581 00000100:00080000:1.0:1775767119.927800:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767119.927802:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767119.927805:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767119.927814:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099017088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767119.927820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767119.927823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767119.927825:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767119.927827:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4581 00000100:00080000:1.0:1775767119.927845:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767119.927848:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767119.927850:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767119.927863:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099017216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767119.927869:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767119.927928:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767119/real 1775767119] req@ffff8b23864fce00 x1862022099016704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767119.927949:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099016704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767119.927960:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767119.927965:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767119.927970:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099016704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767119.927979:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099016832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767119.928007:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767119.928039:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767119/real 1775767119] req@ffff8b23864fd180 x1862022099016832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767119.928053:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099016832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767119.928060:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767119.928064:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767119.928068:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099016832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767119.928102:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099016960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767119.928124:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767119.928136:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767119/real 1775767119] req@ffff8b23864fea00 x1862022099016960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767119.928147:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099016960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767119.928157:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767119.928160:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767119.928164:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099016960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767119.928171:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099017088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767119.928194:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767119.928206:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767119/real 1775767119] req@ffff8b23864fca80 x1862022099017088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767119.928219:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099017088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767119.928228:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767119.928232:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767119.928236:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099017088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767119.928243:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099017216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767119.928311:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767119.928324:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767119/real 1775767119] req@ffff8b23864fe680 x1862022099017216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767119.928337:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099017216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767119.928346:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767119.928350:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767119.928354:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099017216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767120.823733:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767121.847669:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767122.871813:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767123.895709:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767124.919808:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767125.047584:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767125.047597:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767125.047604:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767125.047616:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4586 00000100:00080000:1.0:1775767125.047646:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767125.047655:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767125.047727:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099017344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767125.047766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767125.047771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767125.047774:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767125.047780:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4586 00000100:00080000:1.0:1775767125.047785:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767125.047791:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767125.047806:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767125.047826:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099017472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767125.047841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767125.047846:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099017344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767125.047846:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767125.047849:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767125.047859:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4586 00000100:00080000:1.0:1775767125.047863:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767125.047867:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767125.047872:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767125.047891:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099017600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767125.047907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767125.047911:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767125.047912:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767125.047916:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4586 00000100:00080000:1.0:1775767125.047928:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767125.047931:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767125.047935:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767125.047942:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775767125.047948:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099017728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767125.047957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767125.047961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767125.047963:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767125.047967:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4586 00000100:00080000:1.0:1775767125.047970:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767125.047973:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767125.047977:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767125.047986:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099017856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767125.048003:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767125/real 1775767125] req@ffff8b23864fdc00 x1862022099017344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767125.048041:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099017344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767125.048052:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767125.048056:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767125.048060:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099017344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767125.048105:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099017472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767125.048127:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767125.048138:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767125/real 1775767125] req@ffff8b23864fd500 x1862022099017472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767125.048156:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099017472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767125.048163:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767125.048166:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767125.048169:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099017472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767125.048173:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099017600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767125.048187:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767125.048194:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767125/real 1775767125] req@ffff8b23864ff100 x1862022099017600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767125.048233:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099017600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767125.048258:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767125.048262:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767125.048266:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099017600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767125.048274:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099017728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767125.048318:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767125.048333:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767125/real 1775767125] req@ffff8b23864fe300 x1862022099017728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767125.048344:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099017728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767125.048357:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767125.048359:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767125.048364:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099017728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767125.048372:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099017856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767125.048413:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767125.048425:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767125/real 1775767125] req@ffff8b23864ffb80 x1862022099017856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767125.048440:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099017856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767125.048451:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767125.048455:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767125.048458:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099017856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767125.943609:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767126.967640:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767127.991695:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767129.015739:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767130.039709:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767130.167634:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767130.167652:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767130.167661:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767130.167695:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4591 00000100:00080000:3.0:1775767130.167715:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767130.167726:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767130.167793:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099017984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767130.167820:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767130.167823:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767130.167825:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767130.167828:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4591 00000100:00080000:3.0:1775767130.167831:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767130.167834:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767130.167844:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767130.167855:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099018112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767130.167881:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767130.167885:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767130.167887:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767130.167892:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4591 00000100:00080000:3.0:1775767130.167895:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767130.167899:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767130.167904:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767130.167907:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099017984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767130.167918:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099018240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767130.167930:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767130.167934:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767130.167936:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767130.167940:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4591 00000100:00080000:3.0:1775767130.167944:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767130.167947:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767130.167951:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767130.167978:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099018368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767130.167984:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767130.167986:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767130.167987:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767130.167990:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4591 00000100:00080000:3.0:1775767130.167993:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767130.167995:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767130.167998:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767130.168005:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099018496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767130.168019:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767130.168065:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767130/real 1775767130] req@ffff8b23864f3480 x1862022099017984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767130.168080:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099017984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767130.168093:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767130.168096:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767130.168100:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099017984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767130.168105:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099018112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767130.168123:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767130.168132:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767130/real 1775767130] req@ffff8b23864f2a00 x1862022099018112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767130.168158:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099018112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767130.168167:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767130.168172:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767130.168176:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099018112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767130.168206:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099018240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767130.168223:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767130.168231:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767130/real 1775767130] req@ffff8b23864f0000 x1862022099018240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767130.168239:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099018240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767130.168250:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767130.168253:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767130.168255:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099018240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767130.168261:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099018368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767130.168273:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767130.168279:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767130/real 1775767130] req@ffff8b23864f1180 x1862022099018368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767130.168287:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099018368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767130.168292:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767130.168293:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767130.168295:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099018368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767130.168299:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099018496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767130.168311:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767130.168316:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767130/real 1775767130] req@ffff8b23864f3b80 x1862022099018496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767130.168323:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099018496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767130.168328:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767130.168330:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767130.168332:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099018496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767131.063681:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767132.087765:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767133.111629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767134.135685:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767135.159592:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767135.287609:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767135.287623:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767135.287629:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767135.287643:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4596 00000100:00080000:3.0:1775767135.287658:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767135.287667:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767135.287720:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099018624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767135.287756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767135.287761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767135.287763:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767135.287767:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4596 00000100:00080000:3.0:1775767135.287770:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767135.287775:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767135.287786:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767135.287803:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099018752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767135.287817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767135.287822:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099018624:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767135.287822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767135.287825:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767135.287835:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4596 00000100:00080000:3.0:1775767135.287841:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767135.287846:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767135.287851:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767135.287867:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099018880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767135.287877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767135.287882:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767135.287884:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767135.287889:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4596 00000100:00080000:3.0:1775767135.287894:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767135.287898:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767135.287903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767135.287915:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099019008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767135.287923:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767135.287931:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767135.287935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767135.287937:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767135.287942:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4596 00000100:00080000:3.0:1775767135.287947:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767135.287952:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767135.287957:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767135.287969:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099019136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767135.287988:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767135/real 1775767135] req@ffff8b23864f2d80 x1862022099018624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767135.288024:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099018624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767135.288041:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767135.288046:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767135.288051:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099018624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767135.288080:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099018752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767135.288121:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767135.288137:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767135/real 1775767135] req@ffff8b23864f3100 x1862022099018752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767135.288152:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099018752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767135.288160:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767135.288164:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767135.288168:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099018752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767135.288176:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099018880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767135.288197:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767135.288208:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767135/real 1775767135] req@ffff8b23864f0700 x1862022099018880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767135.288229:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099018880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767135.288238:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767135.288242:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767135.288245:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099018880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767135.288251:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099019008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767135.288276:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767135.288288:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767135/real 1775767135] req@ffff8b23864f2680 x1862022099019008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767135.288301:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099019008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767135.288309:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767135.288313:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767135.288317:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099019008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767135.288323:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099019136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767135.288345:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767135.288357:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767135/real 1775767135] req@ffff8b23864f0380 x1862022099019136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767135.288370:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099019136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767135.288384:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767135.288388:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767135.288391:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099019136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767136.183762:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767137.207695:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767138.231663:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767139.255730:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767140.279662:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767140.407587:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767140.407598:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767140.407603:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767140.407614:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4601 00000100:00080000:3.0:1775767140.407628:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767140.407636:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767140.407683:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099019264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767140.407708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767140.407714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767140.407715:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767140.407718:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4601 00000100:00080000:3.0:1775767140.407720:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767140.407723:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767140.407732:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767140.407742:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099019392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767140.407751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767140.407754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767140.407756:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767140.407759:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4601 00000100:00080000:3.0:1775767140.407761:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767140.407764:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767140.407767:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767140.407779:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099019520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767140.407780:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099019264:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767140.407794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767140.407797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767140.407798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767140.407801:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4601 00000100:00080000:3.0:1775767140.407803:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767140.407805:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767140.407809:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767140.407818:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099019648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767140.407824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767140.407827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767140.407828:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767140.407831:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4601 00000100:00080000:3.0:1775767140.407834:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767140.407836:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767140.407839:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767140.407845:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099019776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767140.407862:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767140.407906:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767140/real 1775767140] req@ffff8b23864f0a80 x1862022099019264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767140.407921:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099019264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767140.407929:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767140.407932:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767140.407936:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099019264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767140.407941:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099019392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767140.407956:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767140.407965:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767140/real 1775767140] req@ffff8b23864f1f80 x1862022099019392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767140.407973:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099019392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767140.407983:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767140.407985:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767140.407987:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099019392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767140.408010:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099019520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767140.408021:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767140.408059:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767140/real 1775767140] req@ffff8b23864f1c00 x1862022099019520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767140.408069:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099019520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767140.408075:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767140.408077:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767140.408080:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099019520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767140.408084:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099019648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767140.408104:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767140.408114:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767140/real 1775767140] req@ffff8b23864f0380 x1862022099019648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767140.408127:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099019648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767140.408135:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767140.408139:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767140.408143:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099019648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767140.408150:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099019776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767140.408173:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767140.408182:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767140/real 1775767140] req@ffff8b23864f2680 x1862022099019776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767140.408190:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099019776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767140.408196:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767140.408198:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767140.408200:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099019776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767141.303656:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767142.327659:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767143.351709:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767144.375645:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767145.399727:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767145.527526:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767145.527538:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767145.527543:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767145.527554:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4606 00000100:00080000:3.0:1775767145.527567:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767145.527574:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767145.527623:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099019904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767145.527653:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767145.527656:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767145.527658:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767145.527661:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4606 00000100:00080000:3.0:1775767145.527664:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767145.527668:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767145.527679:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767145.527692:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099020032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767145.527703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767145.527709:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767145.527712:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767145.527715:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099019904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767145.527721:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4606 00000100:00080000:3.0:1775767145.527728:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767145.527733:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767145.527737:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767145.527749:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099020160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767145.527758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767145.527761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767145.527762:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767145.527766:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4606 00000100:00080000:3.0:1775767145.527769:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767145.527772:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767145.527775:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767145.527783:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099020288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767145.527796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775767145.527798:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767145.527800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767145.527803:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767145.527810:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4606 00000100:00080000:3.0:1775767145.527813:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767145.527817:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767145.527821:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767145.527830:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099020416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767145.527870:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767145/real 1775767145] req@ffff8b23864f0700 x1862022099019904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767145.527888:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099019904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767145.527900:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767145.527905:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767145.527910:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099019904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767145.527937:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099020032:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767145.527962:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767145.527973:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767145/real 1775767145] req@ffff8b23864f3100 x1862022099020032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767145.527982:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099020032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767145.528004:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767145.528007:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767145.528009:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099020032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767145.528026:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099020160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767145.528043:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767145.528051:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767145/real 1775767145] req@ffff8b23864f2d80 x1862022099020160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767145.528059:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099020160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767145.528064:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767145.528067:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767145.528069:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099020160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767145.528076:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099020288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767145.528098:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767145.528109:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767145/real 1775767145] req@ffff8b23864f3b80 x1862022099020288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767145.528129:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099020288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767145.528138:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767145.528142:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767145.528145:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099020288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767145.528152:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099020416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767145.528170:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767145.528177:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767145/real 1775767145] req@ffff8b23864f1180 x1862022099020416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767145.528185:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099020416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767145.528191:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767145.528193:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767145.528195:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099020416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767146.423627:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767147.447747:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767148.471611:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767149.495688:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767150.519621:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767150.647656:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767150.647666:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767150.647671:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767150.647681:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4612 00000100:00080000:2.0:1775767150.647694:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767150.647701:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767150.647762:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099020544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767150.647789:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767150.647792:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767150.647794:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767150.647796:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4612 00000100:00080000:2.0:1775767150.647799:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767150.647802:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767150.647810:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767150.647820:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099020672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767150.647829:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767150.647831:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767150.647832:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767150.647835:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4612 00000100:00080000:2.0:1775767150.647837:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767150.647840:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767150.647843:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767150.647851:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022099020800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767150.647858:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767150.647860:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767150.647862:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767150.647865:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4612 00000100:00080000:2.0:1775767150.647867:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767150.647873:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099020544:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767150.647873:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767150.647887:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767150.647896:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099020928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767150.647902:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767150.647905:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767150.647906:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767150.647909:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4612 00000100:00080000:2.0:1775767150.647911:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767150.647913:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767150.647917:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767150.647942:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099021056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767150.647990:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767150.648033:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767150/real 1775767150] req@ffff8b23b9ea7b80 x1862022099020544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767150.648060:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099020544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767150.648069:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767150.648073:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767150.648076:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099020544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767150.648081:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099020672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767150.648097:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767150.648105:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767150/real 1775767150] req@ffff8b23b9ea5c00 x1862022099020672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767150.648113:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099020672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767150.648118:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767150.648120:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767150.648122:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099020672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767150.648125:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099020800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767150.648137:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767150.648143:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767150/real 1775767150] req@ffff8b23b9ea5500 x1862022099020800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767150.648150:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022099020800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767150.648160:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767150.648162:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767150.648164:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099020800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767150.648189:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099020928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767150.648200:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767150.648206:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767150/real 1775767150] req@ffff8b23b9ea5f80 x1862022099020928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767150.648229:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099020928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767150.648235:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767150.648236:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767150.648238:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099020928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767150.648243:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099021056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767150.648255:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767150.648260:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767150/real 1775767150] req@ffff8b23b9ea6680 x1862022099021056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767150.648267:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099021056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767150.648273:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767150.648274:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767150.648277:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099021056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767151.543686:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767152.567635:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767153.591677:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767154.615699:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767155.639610:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767155.767693:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767155.767707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767155.767713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767155.767724:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4617 00000100:00080000:1.0:1775767155.767754:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767155.767766:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767155.767843:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099021184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767155.767893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767155.767897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767155.767899:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767155.767903:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4617 00000100:00080000:1.0:1775767155.767907:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767155.767910:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767155.767920:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767155.767934:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099021312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767155.767947:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099021184:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767155.767956:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767155.767960:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767155.767961:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767155.767966:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4617 00000100:00080000:1.0:1775767155.767969:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767155.767972:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767155.767976:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767155.767987:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099021440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767155.767995:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767155.767998:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767155.768000:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767155.768003:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4617 00000100:00080000:1.0:1775767155.768007:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767155.768009:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767155.768013:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767155.768018:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775767155.768027:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099021568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767155.768041:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767155.768045:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767155.768047:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767155.768053:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4617 00000100:00080000:1.0:1775767155.768059:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767155.768061:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767155/real 1775767155] req@ffff8b23864ffb80 x1862022099021184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767155.768068:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767155.768081:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767155.768089:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099021184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767155.768100:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1775767155.768104:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099021696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767155.768109:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767155.768118:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099021184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767155.768145:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099021312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767155.768164:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767155.768174:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767155/real 1775767155] req@ffff8b23864fe300 x1862022099021312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767155.768184:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099021312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767155.768191:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767155.768194:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767155.768196:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099021312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767155.768212:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099021440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767155.768227:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767155.768235:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767155/real 1775767155] req@ffff8b23864ff100 x1862022099021440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767155.768245:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099021440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767155.768251:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767155.768253:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767155.768256:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099021440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767155.768260:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099021568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767155.768275:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767155.768282:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767155/real 1775767155] req@ffff8b23864fd500 x1862022099021568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767155.768290:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099021568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767155.768297:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767155.768299:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767155.768301:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099021568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767155.768305:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099021696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767155.768319:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767155.768325:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767155/real 1775767155] req@ffff8b23864fdc00 x1862022099021696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767155.768340:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099021696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767155.768346:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767155.768348:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767155.768350:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099021696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767156.663646:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767157.687725:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767158.711649:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767159.735833:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767160.759614:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767160.887605:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767160.887615:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767160.887620:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767160.887630:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4622 00000100:00080000:3.0:1775767160.887643:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767160.887650:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767160.887693:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099021824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767160.887721:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767160.887725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767160.887726:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767160.887729:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4622 00000100:00080000:3.0:1775767160.887731:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767160.887734:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767160.887746:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767160.887756:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099021952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767160.887766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767160.887769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767160.887770:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767160.887773:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4622 00000100:00080000:3.0:1775767160.887775:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767160.887778:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767160.887784:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767160.887786:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099021824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767160.887794:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099022080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767160.887802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767160.887804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767160.887806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767160.887808:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4622 00000100:00080000:3.0:1775767160.887810:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767160.887813:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767160.887816:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767160.887838:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099022208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767160.887845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767160.887847:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767160.887849:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767160.887852:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4622 00000100:00080000:3.0:1775767160.887855:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767160.887858:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767160.887861:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767160.887869:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099022336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767160.887882:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767160.887925:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767160/real 1775767160] req@ffff8b23864f0000 x1862022099021824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767160.887942:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099021824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767160.887951:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767160.887954:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767160.887957:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099021824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767160.887963:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099021952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767160.887981:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767160.887989:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767160/real 1775767160] req@ffff8b23864f2a00 x1862022099021952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767160.888003:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099021952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767160.888008:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767160.888010:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767160.888012:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099021952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767160.888035:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099022080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767160.888046:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767160.888052:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767160/real 1775767160] req@ffff8b23864f3480 x1862022099022080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767160.888060:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099022080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767160.888065:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767160.888067:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767160.888069:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099022080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767160.888072:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099022208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767160.888102:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767160.888108:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767160/real 1775767160] req@ffff8b23864f1180 x1862022099022208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767160.888116:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099022208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767160.888127:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767160.888130:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767160.888132:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099022208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767160.888136:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099022336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767160.888147:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767160.888153:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767160/real 1775767160] req@ffff8b23864f3b80 x1862022099022336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767160.888161:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099022336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767160.888167:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767160.888169:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767160.888170:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099022336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767161.783643:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767162.807625:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767163.831628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767164.855836:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767165.879739:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767166.007566:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767166.007580:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767166.007585:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767166.007595:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4627 00000100:00080000:2.0:1775767166.007608:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767166.007614:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767166.007667:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099022464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767166.007706:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767166.007709:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767166.007711:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767166.007714:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4627 00000100:00080000:2.0:1775767166.007716:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767166.007718:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767166.007728:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767166.007737:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099022592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767166.007746:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767166.007749:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767166.007750:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767166.007753:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4627 00000100:00080000:2.0:1775767166.007756:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767166.007759:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767166.007762:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767166.007778:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099022464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775767166.007786:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099022720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767166.007798:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767166.007801:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767166.007802:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767166.007805:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4627 00000100:00080000:2.0:1775767166.007808:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767166.007810:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767166.007813:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767166.007822:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099022848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767166.007828:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767166.007831:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767166.007832:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767166.007835:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4627 00000100:00080000:2.0:1775767166.007837:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767166.007839:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767166.007842:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767166.007850:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099022976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767166.007868:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767166.007914:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767166/real 1775767166] req@ffff8b23b9ea6d80 x1862022099022464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767166.007930:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099022464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767166.007939:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767166.007944:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767166.007947:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099022464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767166.007953:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099022592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767166.007969:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767166.007977:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767166/real 1775767166] req@ffff8b23b9ea4700 x1862022099022592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767166.007984:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099022592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767166.007989:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767166.007991:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767166.007993:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099022592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767166.008017:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099022720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767166.008028:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767166.008034:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767166/real 1775767166] req@ffff8b23b9ea6a00 x1862022099022720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767166.008041:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099022720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767166.008046:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767166.008048:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767166.008050:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099022720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767166.008054:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099022848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767166.008065:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767166.008071:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767166/real 1775767166] req@ffff8b23b9ea4000 x1862022099022848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767166.008078:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099022848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767166.008083:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767166.008085:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767166.008086:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099022848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767166.008090:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099022976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767166.008099:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767166.008104:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767166/real 1775767166] req@ffff8b23b9ea5880 x1862022099022976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767166.008110:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099022976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767166.008115:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767166.008117:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767166.008119:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099022976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767166.903756:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767167.927686:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767168.951657:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767169.975610:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767170.999669:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767171.127544:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767171.127557:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767171.127563:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767171.127577:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4632 00000100:00080000:2.0:1775767171.127592:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767171.127610:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767171.127660:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099023104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767171.127699:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767171.127703:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767171.127705:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767171.127710:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4632 00000100:00080000:2.0:1775767171.127714:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767171.127732:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767171.127745:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767171.127760:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099023232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767171.127764:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099023104:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767171.127784:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767171.127788:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767171.127790:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767171.127796:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4632 00000100:00080000:2.0:1775767171.127801:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767171.127806:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767171.127812:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767171.127825:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099023360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767171.127836:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767171.127841:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767171.127843:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775767171.127848:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775767171.127853:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4632 00000100:00080000:2.0:1775767171.127859:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767171.127863:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767171.127870:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767171.127882:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022099023488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767171.127894:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767171.127898:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767171.127900:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767171.127905:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4632 00000100:00080000:2.0:1775767171.127909:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767171.127913:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767171/real 1775767171] req@ffff8b23b9ea4a80 x1862022099023104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767171.127916:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767171.127924:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767171.127942:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099023616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767171.127948:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099023104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767171.127963:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767171.127969:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767171.127975:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099023104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767171.128005:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099023232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767171.128032:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767171.128046:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767171/real 1775767171] req@ffff8b23b9ea7100 x1862022099023232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767171.128060:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099023232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767171.128067:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767171.128069:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767171.128072:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099023232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767171.128076:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099023360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767171.128092:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767171.128100:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767171/real 1775767171] req@ffff8b23b9ea4380 x1862022099023360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767171.128108:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099023360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767171.128118:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767171.128120:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767171.128122:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099023360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767171.128126:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099023488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767171.128137:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767171.128142:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767171/real 1775767171] req@ffff8b23b9ea4e00 x1862022099023488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767171.128149:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022099023488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767171.128154:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767171.128156:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767171.128158:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099023488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767171.128161:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099023616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767171.128170:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767171.128175:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767171/real 1775767171] req@ffff8b23b9ea7480 x1862022099023616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767171.128182:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099023616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767171.128187:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767171.128189:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767171.128192:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099023616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767172.023655:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767173.047694:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767174.071674:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767175.095764:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767176.119652:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767176.247584:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767176.247595:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767176.247600:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767176.247611:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4637 00000100:00080000:2.0:1775767176.247623:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767176.247630:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767176.247702:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099023744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767176.247731:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767176.247734:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767176.247736:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767176.247739:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4637 00000100:00080000:2.0:1775767176.247741:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767176.247743:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767176.247753:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767176.247763:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099023872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767176.247772:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767176.247776:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767176.247779:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767176.247784:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4637 00000100:00080000:2.0:1775767176.247788:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767176.247794:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099023744:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767176.247795:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767176.247804:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767176.247818:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022099024000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767176.247828:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767176.247832:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767176.247834:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767176.247838:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4637 00000100:00080000:2.0:1775767176.247843:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767176.247847:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767176.247852:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767176.247863:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099024128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767176.247873:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767176.247876:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767176.247878:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775767176.247883:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775767176.247887:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4637 00000100:00080000:2.0:1775767176.247892:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767176.247895:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767176.247901:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767176.247913:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099024256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767176.247938:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767176/real 1775767176] req@ffff8b23b9ea6300 x1862022099023744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767176.247955:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099023744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767176.247965:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767176.247969:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767176.247972:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099023744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767176.247978:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099023872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767176.247994:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767176.248003:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767176/real 1775767176] req@ffff8b23b9ea7480 x1862022099023872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767176.248011:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099023872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767176.248016:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767176.248018:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767176.248020:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099023872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767176.248043:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099024000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767176.248054:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767176.248060:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767176/real 1775767176] req@ffff8b23b9ea4e00 x1862022099024000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767176.248068:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022099024000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767176.248073:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767176.248075:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767176.248077:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099024000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767176.248081:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099024128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767176.248092:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767176.248098:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767176/real 1775767176] req@ffff8b23b9ea4380 x1862022099024128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767176.248105:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099024128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767176.248110:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767176.248112:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767176.248113:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099024128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767176.248117:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099024256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767176.248126:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767176.248131:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767176/real 1775767176] req@ffff8b23b9ea7100 x1862022099024256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767176.248143:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099024256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767176.248148:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767176.248149:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767176.248151:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099024256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767177.143589:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767178.167668:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767179.191678:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767180.215692:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767181.239653:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767181.367540:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767181.367557:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767181.367561:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767181.367572:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4642 00000100:00080000:2.0:1775767181.367583:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767181.367590:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767181.367672:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099024384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767181.367700:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767181.367706:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767181.367708:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767181.367711:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4642 00000100:00080000:2.0:1775767181.367714:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767181.367717:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767181.367728:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767181.367739:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099024512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767181.367751:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767181.367757:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775767181.367759:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099024384:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767181.367759:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767181.367767:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4642 00000100:00080000:2.0:1775767181.367774:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767181.367778:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767181.367784:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767181.367799:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099024640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767181.367824:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767181.367829:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767181.367832:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767181.367854:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4642 00000100:00080000:2.0:1775767181.367860:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767181.367865:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767181.367873:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767181.367887:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099024768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767181.367896:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767181.367899:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767181.367900:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767181.367903:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4642 00000100:00080000:2.0:1775767181.367907:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767181.367911:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099024512:192.168.202.128@tcp:38:kworker.0 00000100:00080000:2.0:1775767181.367922:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767181.367928:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767181.367941:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099024896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767181.367952:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099024640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767181.367975:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099024768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767181.367998:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099024896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767182.263655:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767183.287609:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767184.311664:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767184.439852:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775767184.439918:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767184.439944:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767184.439998:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767184.440009:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767184.440018:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767184.440028:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775767184.440076:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767181/real 1775767184] req@ffff8b23b9ea4700 x1862022099024896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767184.440110:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099024896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767184.440125:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767184.440133:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767184.440200:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099024896:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767184.440210:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767181/real 1775767184] req@ffff8b23b9ea5880 x1862022099024512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767184.440225:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099024512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767184.440235:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767184.440240:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767184.440245:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099024512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767184.440254:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767181/real 1775767184] req@ffff8b23b9ea4000 x1862022099024640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767184.440268:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099024640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767184.440277:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767184.440281:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767184.440285:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099024640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767184.440294:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767181/real 1775767184] req@ffff8b23b9ea6a00 x1862022099024768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767184.440309:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099024768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767184.440318:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767184.440322:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767184.440327:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099024768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767184.440335:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767181/real 1775767184] req@ffff8b23b9ea4a80 x1862022099024384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767184.440349:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099024384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767184.440358:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767184.440362:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767184.440365:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099024384:192.168.202.128@tcp:250:kworker.0 00010000:00010000:0.0:1775767185.335612:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767186.359657:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767186.487623:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767186.487635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767186.487640:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767186.487653:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4647 00000100:00080000:1.0:1775767186.487667:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767186.487675:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767186.487725:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099025024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767186.487769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767186.487777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767186.487779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767186.487784:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4647 00000100:00080000:1.0:1775767186.487788:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767186.487794:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767186.487809:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767186.487827:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099025152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767186.487835:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099025024:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767186.487844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767186.487848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767186.487850:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767186.487856:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4647 00000100:00080000:1.0:1775767186.487861:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767186.487866:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767186.487871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767186.487887:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099025280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767186.487899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767186.487903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767186.487905:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767186.487910:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4647 00000100:00080000:1.0:1775767186.487926:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767186.487930:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767186.487938:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767186.487944:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775767186.487960:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099025408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767186.487972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767186.487975:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767186.487977:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767186.487981:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4647 00000100:00080000:1.0:1775767186.487988:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767186.487997:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767186.488004:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767186.488011:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767186/real 1775767186] req@ffff8b23864fe680 x1862022099025024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767186.488023:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099025536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767186.488042:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099025024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767186.488058:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767186.488065:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767186.488070:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099025024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767186.488103:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099025152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767186.488129:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767186.488143:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767186/real 1775767186] req@ffff8b23864fca80 x1862022099025152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767186.488165:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099025152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767186.488174:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767186.488178:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767186.488182:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099025152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767186.488189:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099025280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767186.488216:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767186.488231:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767186/real 1775767186] req@ffff8b23864fea00 x1862022099025280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767186.488242:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099025280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767186.488251:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767186.488255:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767186.488259:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099025280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767186.488265:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099025408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767186.488291:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767186.488302:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767186/real 1775767186] req@ffff8b23864fd180 x1862022099025408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767186.488314:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099025408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767186.488328:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767186.488332:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767186.488335:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099025408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767186.488341:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099025536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767186.488363:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767186.488374:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767186/real 1775767186] req@ffff8b23864fce00 x1862022099025536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767186.488385:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099025536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767186.488395:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767186.488398:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767186.488401:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099025536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767187.383594:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767188.407686:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767189.431628:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767190.455708:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767191.479581:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767191.607572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767191.607580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767191.607584:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767191.607593:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4653 00000100:00080000:1.0:1775767191.607603:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767191.607609:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767191.607646:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099025664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767191.607670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767191.607673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767191.607675:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767191.607678:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4653 00000100:00080000:1.0:1775767191.607680:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767191.607682:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767191.607689:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767191.607700:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099025792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767191.607713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767191.607718:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099025664:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767191.607719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767191.607723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767191.607729:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4653 00000100:00080000:1.0:1775767191.607734:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767191.607738:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767191.607745:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767191.607761:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099025920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767191.607774:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775767191.607783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767191.607786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767191.607788:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767191.607793:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4653 00000100:00080000:1.0:1775767191.607796:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767191.607799:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767191.607804:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767191.607814:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767191/real 1775767191] req@ffff8b23864fc000 x1862022099025664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767191.607823:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099026048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767191.607837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767191.607840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767191.607841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767191.607844:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099025664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767191.607847:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4653 00000100:00080000:1.0:1775767191.607853:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767191.607858:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767191.607864:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775767191.607874:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1775767191.607877:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099026176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767191.607881:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767191.607889:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099025664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767191.607915:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099025792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767191.607933:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767191.607944:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767191/real 1775767191] req@ffff8b23864fc380 x1862022099025792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767191.607957:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099025792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767191.607965:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767191.607968:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767191.607972:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099025792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767191.607978:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099025920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767191.607999:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767191.608010:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767191/real 1775767191] req@ffff8b23864fdf80 x1862022099025920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767191.608018:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099025920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767191.608024:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767191.608026:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767191.608028:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099025920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767191.608032:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099026048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767191.608047:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767191.608053:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767191/real 1775767191] req@ffff8b23864ff800 x1862022099026048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767191.608060:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099026048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767191.608065:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767191.608067:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767191.608069:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099026048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767191.608072:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099026176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767191.608082:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767191.608087:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767191/real 1775767191] req@ffff8b23864fed80 x1862022099026176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767191.608094:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099026176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767191.608098:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767191.608100:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767191.608102:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099026176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767192.503681:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767193.527610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767194.551636:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767195.575659:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767196.599640:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767196.727597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767196.727608:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767196.727613:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767196.727623:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4658 00000100:00080000:3.0:1775767196.727636:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767196.727642:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767196.727686:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099026304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767196.727712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767196.727715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767196.727717:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767196.727720:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4658 00000100:00080000:3.0:1775767196.727722:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767196.727725:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767196.727733:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767196.727745:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099026432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767196.727760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767196.727763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767196.727765:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767196.727771:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099026304:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767196.727771:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4658 00000100:00080000:3.0:1775767196.727775:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767196.727782:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767196.727785:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767196.727794:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099026560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767196.727800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767196.727803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767196.727805:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767196.727808:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4658 00000100:00080000:3.0:1775767196.727810:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767196.727812:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767196.727815:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767196.727823:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099026688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767196.727829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767196.727832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767196.727833:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767196.727835:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4658 00000100:00080000:3.0:1775767196.727839:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767196.727842:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767196.727845:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767196.727852:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767196.727855:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099026816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767196.727900:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767196/real 1775767196] req@ffff8b23864f2d80 x1862022099026304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767196.727916:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099026304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767196.727924:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767196.727928:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767196.727931:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099026304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767196.727953:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099026432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767196.727968:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767196.727976:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767196/real 1775767196] req@ffff8b23864f3100 x1862022099026432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767196.727984:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099026432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767196.727990:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767196.727992:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767196.727994:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099026432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767196.727998:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099026560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767196.728008:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767196.728013:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767196/real 1775767196] req@ffff8b23864f0700 x1862022099026560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767196.728025:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099026560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767196.728030:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767196.728032:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767196.728034:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099026560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767196.728037:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099026688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767196.728048:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767196.728053:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767196/real 1775767196] req@ffff8b23864f2680 x1862022099026688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767196.728060:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099026688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767196.728065:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767196.728067:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767196.728069:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099026688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767196.728072:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099026816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767196.728081:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767196.728086:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767196/real 1775767196] req@ffff8b23864f0380 x1862022099026816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767196.728092:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099026816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767196.728102:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767196.728104:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767196.728106:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099026816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767197.623669:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767198.647616:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767199.671650:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767200.695665:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767201.719646:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767201.847836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767201.847847:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767201.847852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767201.847863:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4663 00000100:00080000:3.0:1775767201.847877:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767201.847883:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767201.847931:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099026944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767201.847959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767201.847962:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767201.847963:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767201.847966:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4663 00000100:00080000:3.0:1775767201.847969:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767201.847971:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767201.847980:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767201.847991:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099027072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767201.847999:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767201.848004:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767201.848006:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767201.848009:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4663 00000100:00080000:3.0:1775767201.848012:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767201.848015:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767201.848017:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767201.848027:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099026944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767201.848027:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099027200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767201.848041:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767201.848044:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767201.848045:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767201.848048:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4663 00000100:00080000:3.0:1775767201.848050:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767201.848052:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767201.848055:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767201.848064:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099027328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767201.848070:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767201.848072:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767201.848074:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767201.848076:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4663 00000100:00080000:3.0:1775767201.848079:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767201.848082:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767201.848085:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767201.848096:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099027456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767201.848111:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767201.848162:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767201/real 1775767201] req@ffff8b23864f1c00 x1862022099026944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767201.848183:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099026944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767201.848196:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767201.848201:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767201.848205:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099026944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767201.848212:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099027072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767201.848237:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767201.848250:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767201/real 1775767201] req@ffff8b23864f1f80 x1862022099027072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767201.848264:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099027072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767201.848278:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767201.848282:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767201.848285:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099027072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767201.848314:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099027200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767201.848335:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767201.848346:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767201/real 1775767201] req@ffff8b23864f0a80 x1862022099027200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767201.848358:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099027200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767201.848367:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767201.848370:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767201.848373:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099027200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767201.848379:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099027328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767201.848399:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767201.848429:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767201/real 1775767201] req@ffff8b23864f0380 x1862022099027328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767201.848443:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099027328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767201.848495:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767201.848500:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767201.848505:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099027328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767201.848557:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099027456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767201.848584:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767201.848600:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767201/real 1775767201] req@ffff8b23864f2680 x1862022099027456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767201.848614:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099027456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767201.848621:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767201.848623:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767201.848625:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099027456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767202.743624:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767203.767704:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767204.791599:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767205.815712:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767206.839621:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767206.967619:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767206.967627:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767206.967631:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767206.967639:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4668 00000100:00080000:2.0:1775767206.967649:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767206.967655:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767206.967690:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099027584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767206.967719:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767206.967722:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767206.967723:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767206.967726:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4668 00000100:00080000:2.0:1775767206.967729:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767206.967731:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767206.967738:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767206.967746:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099027712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767206.967755:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767206.967758:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767206.967760:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767206.967764:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4668 00000100:00080000:2.0:1775767206.967766:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767206.967769:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767206.967772:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767206.967777:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099027840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767206.967784:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767206.967786:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767206.967788:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767206.967791:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4668 00000100:00080000:2.0:1775767206.967795:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767206.967802:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767206.967803:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099027584:192.168.202.128@tcp:250:kworker.0 00000008:00100000:2.0:1775767206.967805:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767206.967816:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022099027968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767206.967826:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767206.967828:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767206.967829:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767206.967832:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4668 00000100:00080000:2.0:1775767206.967833:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767206.967836:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767206.967840:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767206.967847:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099028096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767206.967906:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767206.967963:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767206/real 1775767206] req@ffff8b23b9ea6d80 x1862022099027584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767206.967986:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099027584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767206.967997:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767206.968002:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767206.968006:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099027584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767206.968015:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099027712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767206.968041:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767206.968054:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767206/real 1775767206] req@ffff8b23b9ea6680 x1862022099027712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767206.968067:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099027712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767206.968076:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767206.968079:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767206.968083:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099027712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767206.968114:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099027840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767206.968136:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767206.968147:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767206/real 1775767206] req@ffff8b23b9ea5f80 x1862022099027840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767206.968159:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099027840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767206.968166:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767206.968170:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767206.968173:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099027840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767206.968179:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099027968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767206.968198:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767206.968208:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767206/real 1775767206] req@ffff8b23b9ea5500 x1862022099027968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767206.968227:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022099027968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767206.968235:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767206.968240:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767206.968245:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099027968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767206.968254:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099028096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767206.968273:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767206.968283:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767206/real 1775767206] req@ffff8b23b9ea5c00 x1862022099028096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767206.968295:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099028096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767206.968303:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767206.968306:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767206.968309:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099028096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767207.863661:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767208.887708:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767209.911676:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767210.935653:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767211.959705:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767212.087542:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767212.087552:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767212.087557:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767212.087567:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4673 00000100:00080000:2.0:1775767212.087580:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767212.087587:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767212.087632:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099028224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767212.087660:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767212.087664:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767212.087666:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767212.087669:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4673 00000100:00080000:2.0:1775767212.087671:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767212.087673:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767212.087684:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767212.087693:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099028352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767212.087701:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767212.087705:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767212.087714:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767212.087717:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4673 00000100:00080000:2.0:1775767212.087720:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767212.087723:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767212.087727:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767212.087731:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099028224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775767212.087738:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022099028480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767212.087746:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767212.087749:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767212.087750:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767212.087754:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4673 00000100:00080000:2.0:1775767212.087757:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767212.087759:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767212.087762:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767212.087770:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099028608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767212.087776:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767212.087778:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767212.087780:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767212.087783:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4673 00000100:00080000:2.0:1775767212.087785:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767212.087787:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767212.087790:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767212.087797:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099028736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767212.087824:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767212.087875:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767212/real 1775767212] req@ffff8b23b9ea7b80 x1862022099028224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767212.087907:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099028224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767212.087918:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767212.087923:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767212.087928:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099028224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767212.087936:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099028352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767212.087961:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767212.087975:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767212/real 1775767212] req@ffff8b23b9ea5c00 x1862022099028352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767212.087988:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099028352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767212.087997:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767212.088001:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767212.088004:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099028352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767212.088032:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099028480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767212.088051:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767212.088062:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767212/real 1775767212] req@ffff8b23b9ea5500 x1862022099028480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767212.088075:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022099028480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767212.088090:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767212.088093:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767212.088095:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099028480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767212.088099:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099028608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767212.088116:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767212.088123:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767212/real 1775767212] req@ffff8b23b9ea5f80 x1862022099028608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767212.088131:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099028608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767212.088136:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767212.088139:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767212.088140:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099028608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767212.088144:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099028736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767212.088153:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767212.088158:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767212/real 1775767212] req@ffff8b23b9ea6680 x1862022099028736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767212.088165:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099028736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767212.088170:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767212.088171:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767212.088174:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099028736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767212.983606:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767214.007656:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767215.031680:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767216.055707:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767217.079634:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767217.207571:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767217.207582:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767217.207586:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767217.207597:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4678 00000100:00080000:2.0:1775767217.207610:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767217.207617:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767217.207662:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099028864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767217.207690:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767217.207693:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767217.207695:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767217.207698:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4678 00000100:00080000:2.0:1775767217.207700:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767217.207702:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767217.207712:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767217.207720:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099028992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767217.207735:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767217.207738:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767217.207739:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767217.207743:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4678 00000100:00080000:2.0:1775767217.207745:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767217.207747:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767217.207751:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767217.207756:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099028864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775767217.207762:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099029120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767217.207769:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767217.207772:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767217.207773:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767217.207776:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4678 00000100:00080000:2.0:1775767217.207779:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767217.207781:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767217.207783:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767217.207790:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099029248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767217.207797:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767217.207800:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767217.207801:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767217.207803:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4678 00000100:00080000:2.0:1775767217.207806:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767217.207809:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767217.207812:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767217.207820:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099029376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767217.207838:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767217.207880:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767217/real 1775767217] req@ffff8b23b9ea6d80 x1862022099028864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767217.207896:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099028864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767217.207906:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767217.207910:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767217.207913:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099028864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767217.207919:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099028992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767217.207935:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767217.207943:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767217/real 1775767217] req@ffff8b23b9ea4a80 x1862022099028992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767217.207951:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099028992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767217.207956:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767217.207958:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767217.207960:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099028992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767217.207983:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099029120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767217.207995:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767217.208001:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767217/real 1775767217] req@ffff8b23b9ea6a00 x1862022099029120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767217.208009:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099029120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767217.208014:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767217.208016:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767217.208018:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099029120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767217.208022:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099029248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767217.208033:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767217.208039:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767217/real 1775767217] req@ffff8b23b9ea4000 x1862022099029248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767217.208046:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099029248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767217.208051:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767217.208053:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767217.208055:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099029248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767217.208058:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099029376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767217.208067:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767217.208072:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767217/real 1775767217] req@ffff8b23b9ea5880 x1862022099029376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767217.208084:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099029376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767217.208089:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767217.208091:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767217.208093:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099029376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767218.103664:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767219.127623:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767220.151606:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767221.175630:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767222.199693:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767222.327660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767222.327672:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767222.327677:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767222.327688:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4683 00000100:00080000:0.0:1775767222.327702:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767222.327709:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767222.327758:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099029504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767222.327793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767222.327798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767222.327801:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767222.327806:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4683 00000100:00080000:0.0:1775767222.327811:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767222.327816:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767222.327829:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767222.327846:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099029632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767222.327855:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099029504:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767222.327860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767222.327864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767222.327866:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767222.327871:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4683 00000100:00080000:0.0:1775767222.327875:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767222.327879:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767222.327885:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767222.327895:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099029760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767222.327921:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767222.327925:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767222.327927:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767222.327932:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4683 00000100:00080000:0.0:1775767222.327948:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767222.327953:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775767222.327958:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:0.0:1775767222.327959:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767222.327974:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099029888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767222.327985:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767222.327989:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767222.327992:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767222.327997:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4683 00000100:00080000:0.0:1775767222.328001:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767222.328006:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767222.328012:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767222.328019:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767222/real 1775767222] req@ffff8b2382daed80 x1862022099029504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767222.328024:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099030016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767222.328046:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099029504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767222.328061:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767222.328067:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767222.328073:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099029504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767222.328107:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099029632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767222.328148:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767222.328159:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767222/real 1775767222] req@ffff8b2382daea00 x1862022099029632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767222.328169:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099029632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767222.328176:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767222.328179:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767222.328182:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099029632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767222.328186:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099029760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767222.328200:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767222.328207:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767222/real 1775767222] req@ffff8b2382dad500 x1862022099029760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767222.328235:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099029760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767222.328242:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767222.328245:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767222.328247:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099029760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767222.328252:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099029888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767222.328285:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767222.328291:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767222/real 1775767222] req@ffff8b2382dae680 x1862022099029888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767222.328299:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099029888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767222.328304:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767222.328305:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767222.328307:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099029888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767222.328311:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099030016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767222.328321:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767222.328326:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767222/real 1775767222] req@ffff8b2382dac380 x1862022099030016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767222.328338:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099030016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767222.328344:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767222.328345:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767222.328347:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099030016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767223.223619:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767224.247686:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767225.271680:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767226.295646:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767227.319676:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767227.447544:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767227.447554:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767227.447559:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767227.447569:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4688 00000100:00080000:0.0:1775767227.447582:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767227.447589:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767227.447634:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022099030144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767227.447678:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767227.447682:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767227.447684:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767227.447694:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4688 00000100:00080000:0.0:1775767227.447698:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767227.447703:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767227.447715:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767227.447730:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022099030272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767227.447742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767227.447746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767227.447748:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767227.447748:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099030144:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767227.447756:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4688 00000100:00080000:0.0:1775767227.447761:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767227.447765:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767227.447771:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767227.447787:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099030400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767227.447798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767227.447802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767227.447805:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767227.447809:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4688 00000100:00080000:0.0:1775767227.447813:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767227.447818:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767227.447823:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767227.447838:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022099030528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767227.447839:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775767227.447852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767227.447856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767227.447858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767227.447862:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4688 00000100:00080000:0.0:1775767227.447868:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767227.447872:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767227.447877:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767227.447893:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099030656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767227.447901:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767227/real 1775767227] req@ffff8b2382dac000 x1862022099030144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767227.447923:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022099030144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767227.447935:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767227.447940:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767227.447945:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099030144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767227.447979:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099030272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767227.448006:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767227.448025:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767227/real 1775767227] req@ffff8b2382daf100 x1862022099030272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767227.448037:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022099030272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767227.448051:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767227.448054:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767227.448058:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099030272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767227.448065:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099030400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767227.448083:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767227.448093:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767227/real 1775767227] req@ffff8b2382daca80 x1862022099030400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767227.448104:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099030400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767227.448113:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767227.448117:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767227.448120:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099030400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767227.448126:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099030528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767227.448149:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767227.448161:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767227/real 1775767227] req@ffff8b2382dad880 x1862022099030528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767227.448188:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022099030528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767227.448194:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767227.448196:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767227.448199:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099030528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767227.448204:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099030656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767227.448228:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767227.448235:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767227/real 1775767227] req@ffff8b2382dac380 x1862022099030656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767227.448243:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099030656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767227.448248:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767227.448251:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767227.448253:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099030656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767228.343631:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767229.367730:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767230.391678:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767231.415695:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767232.439693:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767232.567582:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767232.567596:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767232.567602:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767232.567617:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4693 00000100:00080000:0.0:1775767232.567632:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767232.567641:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767232.567686:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099030784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767232.567716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767232.567719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767232.567721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767232.567724:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4693 00000100:00080000:0.0:1775767232.567726:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767232.567729:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767232.567737:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767232.567746:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099030912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767232.567757:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767232.567760:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767232.567761:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767232.567764:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4693 00000100:00080000:0.0:1775767232.567767:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767232.567769:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767232.567773:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767232.567779:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099030784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767232.567786:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099031040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767232.567794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767232.567797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767232.567798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767232.567800:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4693 00000100:00080000:0.0:1775767232.567802:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767232.567805:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767232.567808:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767232.567816:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099031168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767232.567822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767232.567825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767232.567826:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767232.567829:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4693 00000100:00080000:0.0:1775767232.567832:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767232.567835:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767232.567838:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767232.567845:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099031296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767232.567869:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767232.567931:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767232/real 1775767232] req@ffff8b2382dae680 x1862022099030784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767232.567955:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099030784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767232.567968:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767232.567973:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767232.567978:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099030784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767232.567988:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099030912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767232.568014:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767232.568029:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767232/real 1775767232] req@ffff8b2382dad500 x1862022099030912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767232.568044:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099030912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767232.568053:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767232.568056:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767232.568060:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099030912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767232.568095:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099031040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767232.568135:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767232.568150:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767232/real 1775767232] req@ffff8b2382daea00 x1862022099031040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767232.568164:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099031040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767232.568173:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767232.568177:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767232.568181:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099031040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767232.568188:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099031168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767232.568210:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767232.568222:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767232/real 1775767232] req@ffff8b2382daed80 x1862022099031168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767232.568236:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099031168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767232.568246:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767232.568249:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767232.568253:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099031168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767232.568260:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099031296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767232.568283:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767232.568294:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767232/real 1775767232] req@ffff8b2382daf480 x1862022099031296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767232.568307:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099031296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767232.568316:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767232.568320:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767232.568324:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099031296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767233.463670:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767234.487649:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767235.511651:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767236.535711:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767237.559661:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767237.687598:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767237.687609:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767237.687614:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767237.687624:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4699 00000100:00080000:0.0:1775767237.687636:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767237.687643:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767237.687687:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022099031424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767237.687714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767237.687717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767237.687719:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767237.687722:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4699 00000100:00080000:0.0:1775767237.687724:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767237.687726:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767237.687735:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767237.687744:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022099031552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767237.687757:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767237.687759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767237.687761:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767237.687765:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4699 00000100:00080000:0.0:1775767237.687767:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767237.687771:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767237.687774:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767237.687786:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022099031680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767237.687792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767237.687794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767237.687796:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767237.687801:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4699 00000100:00100000:3.0:1775767237.687802:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099031424:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767237.687803:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767237.687809:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767237.687815:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767237.687821:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022099031808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767237.687827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767237.687829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767237.687831:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767237.687834:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4699 00000100:00080000:0.0:1775767237.687837:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767237.687840:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767237.687843:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767237.687852:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099031936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767237.687888:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767237.687933:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767237/real 1775767237] req@ffff8b2382dac700 x1862022099031424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767237.687949:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022099031424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767237.687958:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767237.687962:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767237.687965:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099031424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767237.687972:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099031552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767237.687987:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767237.687995:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767237/real 1775767237] req@ffff8b2382dace00 x1862022099031552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767237.688003:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022099031552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767237.688008:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767237.688010:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767237.688012:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099031552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767237.688049:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099031680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767237.688063:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767237.688095:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767237/real 1775767237] req@ffff8b2382dadf80 x1862022099031680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767237.688110:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022099031680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767237.688119:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767237.688123:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767237.688126:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099031680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767237.688133:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099031808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767237.688157:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767237.688167:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767237/real 1775767237] req@ffff8b2382dafb80 x1862022099031808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767237.688176:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022099031808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767237.688182:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767237.688184:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767237.688186:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099031808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767237.688190:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099031936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767237.688203:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767237.688209:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767237/real 1775767237] req@ffff8b2382daf480 x1862022099031936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767237.688216:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099031936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767237.688221:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767237.688223:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767237.688225:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099031936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767238.583616:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767239.607745:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767240.631700:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767241.655630:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767242.679693:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767242.807596:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767242.807610:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767242.807618:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767242.807631:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4704 00000100:00080000:2.0:1775767242.807647:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767242.807656:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767242.807702:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099032064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767242.807729:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767242.807732:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767242.807734:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767242.807737:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4704 00000100:00080000:2.0:1775767242.807739:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767242.807742:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767242.807752:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767242.807762:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099032192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767242.807771:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099032064:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767242.807774:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767242.807784:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767242.807785:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767242.807789:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4704 00000100:00080000:2.0:1775767242.807792:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767242.807794:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767242.807797:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767242.807806:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099032320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767242.807812:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767242.807814:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767242.807816:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767242.807819:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4704 00000100:00080000:2.0:1775767242.807822:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767242.807824:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767242.807827:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767242.807834:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022099032448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767242.807839:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767242.807841:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767242.807843:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767242.807845:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4704 00000100:00080000:2.0:1775767242.807850:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767242.807854:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767242.807860:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767242.807865:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775767242.807874:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099032576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767242.807931:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767242/real 1775767242] req@ffff8b23b9ea4700 x1862022099032064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767242.807956:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099032064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767242.807977:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767242.807982:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767242.807987:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099032064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767242.808017:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099032192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767242.808077:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767242.808090:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767242/real 1775767242] req@ffff8b23b9ea7100 x1862022099032192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767242.808104:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099032192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767242.808114:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767242.808117:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767242.808121:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099032192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767242.808128:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099032320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767242.808148:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767242.808159:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767242/real 1775767242] req@ffff8b23b9ea4380 x1862022099032320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767242.808171:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099032320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767242.808180:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767242.808184:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767242.808187:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099032320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767242.808195:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099032448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767242.808217:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767242.808228:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767242/real 1775767242] req@ffff8b23b9ea4e00 x1862022099032448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767242.808242:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022099032448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767242.808251:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767242.808254:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767242.808257:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099032448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767242.808263:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099032576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767242.808284:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767242.808295:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767242/real 1775767242] req@ffff8b23b9ea7480 x1862022099032576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767242.808307:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099032576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767242.808316:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767242.808319:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767242.808322:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099032576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767243.703643:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767244.727708:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767245.751627:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767246.775710:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767247.799630:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767247.927581:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767247.927603:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767247.927608:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767247.927618:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4709 00000100:00080000:2.0:1775767247.927631:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767247.927638:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767247.927696:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099032704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767247.927724:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767247.927726:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767247.927728:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767247.927731:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4709 00000100:00080000:2.0:1775767247.927733:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767247.927736:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767247.927757:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767247.927767:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099032832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767247.927777:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767247.927779:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767247.927781:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767247.927784:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4709 00000100:00080000:2.0:1775767247.927788:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767247.927793:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767247.927793:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099032704:192.168.202.128@tcp:250:kworker.0 00000008:00100000:2.0:1775767247.927800:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767247.927808:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022099032960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767247.927815:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767247.927817:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767247.927819:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767247.927822:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4709 00000100:00080000:2.0:1775767247.927824:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767247.927826:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767247.927830:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767247.927838:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099033088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767247.927844:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767247.927846:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767247.927848:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767247.927850:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4709 00000100:00080000:2.0:1775767247.927854:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767247.927867:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767247.927870:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767247.927877:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099033216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767247.927931:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099032832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767247.928002:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099032960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767247.928028:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099033088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767247.928042:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099033216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767248.823699:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767249.847650:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767250.871647:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767250.999799:0:2363:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775767250.999855:0:2363:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767250.999875:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767250.999933:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767250.999944:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767250.999986:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767250.999995:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:3.0:1775767251.000109:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767247/real 1775767250] req@ffff8b23b9ea4e00 x1862022099032960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767251.000141:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022099032960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767251.000157:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767251.000164:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767251.000169:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099032960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767251.000180:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767247/real 1775767250] req@ffff8b23b9ea4380 x1862022099033088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767251.000193:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099033088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767251.000202:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767251.000206:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767251.000210:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099033088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767251.000217:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767247/real 1775767251] req@ffff8b23b9ea7100 x1862022099033216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767251.000229:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099033216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767251.000238:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767251.000242:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767251.000245:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099033216:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767251.000252:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767247/real 1775767250] req@ffff8b23b9ea6300 x1862022099032704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767251.000264:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099032704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767251.000272:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767251.000276:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767251.000279:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099032704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767251.000287:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767247/real 1775767250] req@ffff8b23b9ea7480 x1862022099032832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767251.000299:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099032832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767251.000307:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767251.000312:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767251.000316:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099032832:192.168.202.128@tcp:38:kworker.0 00010000:00010000:1.0:1775767251.895648:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767252.919676:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767253.047636:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767253.047650:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767253.047655:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767253.047667:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4714 00000100:00080000:3.0:1775767253.047682:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767253.047691:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767253.047738:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099033344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767253.047791:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767253.047796:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767253.047798:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767253.047804:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4714 00000100:00080000:3.0:1775767253.047808:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767253.047813:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767253.047826:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767253.047841:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099033472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767253.047855:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767253.047858:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767253.047860:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767253.047865:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4714 00000100:00080000:3.0:1775767253.047869:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767253.047873:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767253.047878:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767253.047890:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099033600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767253.047900:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767253.047905:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767253.047907:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767253.047911:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4714 00000100:00080000:3.0:1775767253.047915:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767253.047918:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767253.047923:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767253.047955:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099033728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767253.047967:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767253.047970:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767253.047973:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767253.047978:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099033344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767253.047979:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4714 00000100:00080000:3.0:1775767253.047987:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767253.047992:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767253.047998:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767253.048013:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099033856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767253.048075:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767253.048131:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767253/real 1775767253] req@ffff8b23864f0700 x1862022099033344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767253.048154:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099033344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767253.048167:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767253.048173:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767253.048178:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099033344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767253.048189:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099033472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767253.048216:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767253.048231:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767253/real 1775767253] req@ffff8b23864f3100 x1862022099033472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767253.048245:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099033472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767253.048254:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767253.048258:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767253.048262:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099033472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767253.048268:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099033600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767253.048299:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767253.048312:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767253/real 1775767253] req@ffff8b23864f2d80 x1862022099033600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767253.048324:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099033600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767253.048333:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767253.048336:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767253.048340:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099033600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767253.048375:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099033728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767253.048398:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767253.048426:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767253/real 1775767253] req@ffff8b23864f3b80 x1862022099033728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767253.048438:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099033728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767253.048446:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767253.048449:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767253.048496:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099033728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767253.048503:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099033856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767253.048524:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767253.048535:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767253/real 1775767253] req@ffff8b23864f1180 x1862022099033856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767253.048548:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099033856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767253.048557:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767253.048561:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767253.048565:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099033856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767253.943622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767254.967704:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767255.991587:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767257.015618:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767258.039680:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767258.167632:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767258.167644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767258.167650:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767258.167673:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4719 00000100:00080000:0.0:1775767258.167698:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767258.167722:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767258.167776:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099033984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767258.167816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767258.167821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767258.167823:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767258.167829:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4719 00000100:00080000:0.0:1775767258.167835:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767258.167841:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767258.167865:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767258.167873:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099033984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767258.167901:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099034112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767258.167914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767258.167917:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767258.167919:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767258.167923:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4719 00000100:00080000:0.0:1775767258.167927:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767258.167931:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767258.167936:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767258.167947:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099034240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767258.167955:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767258.167959:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767258.167960:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767258.167963:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4719 00000100:00080000:0.0:1775767258.167967:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767258.167973:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775767258.167975:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:0.0:1775767258.167978:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767258.167990:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099034368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767258.167999:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767258.168002:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767258.168004:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767258.168006:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4719 00000100:00080000:0.0:1775767258.168009:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767258.168011:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767258.168015:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767258.168024:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099034496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767258.168041:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767258/real 1775767258] req@ffff8b2382daed80 x1862022099033984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767258.168067:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099033984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767258.168083:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767258.168087:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767258.168091:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099033984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767258.168118:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099034112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767258.168140:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767258.168151:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767258/real 1775767258] req@ffff8b2382daea00 x1862022099034112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767258.168161:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099034112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767258.168168:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767258.168171:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767258.168173:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099034112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767258.168178:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099034240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767258.168191:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767258.168198:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767258/real 1775767258] req@ffff8b2382dad500 x1862022099034240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767258.168207:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099034240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767258.168213:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767258.168216:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767258.168218:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099034240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767258.168223:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099034368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767258.168236:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767258.168242:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767258/real 1775767258] req@ffff8b2382dae680 x1862022099034368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767258.168253:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099034368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767258.168262:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767258.168266:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767258.168270:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099034368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767258.168277:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099034496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767258.168301:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767258.168313:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767258/real 1775767258] req@ffff8b2382dac380 x1862022099034496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767258.168327:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099034496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767258.168338:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767258.168342:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767258.168345:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099034496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767259.063669:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767260.087727:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767261.111682:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767262.135759:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767263.159634:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767263.287543:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767263.287554:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767263.287559:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767263.287570:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4724 00000100:00080000:0.0:1775767263.287583:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767263.287590:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767263.287643:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022099034624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767263.287667:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767263.287670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767263.287672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767263.287675:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4724 00000100:00080000:0.0:1775767263.287677:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767263.287680:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767263.287690:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767263.287707:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099034752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767263.287716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767263.287719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767263.287720:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767263.287724:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4724 00000100:00080000:0.0:1775767263.287728:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767263.287734:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767263.287739:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767263.287755:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022099034880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767263.287757:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099034624:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767263.287774:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767263.287778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767263.287780:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767263.287785:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4724 00000100:00080000:0.0:1775767263.287791:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767263.287794:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767263.287800:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767263.287814:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022099035008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767263.287824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767263.287828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767263.287846:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767263.287852:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4724 00000100:00080000:0.0:1775767263.287856:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767263.287861:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767263.287866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767263.287873:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767263.287882:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099035136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767263.287933:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767263/real 1775767263] req@ffff8b2382dad880 x1862022099034624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767263.287957:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022099034624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767263.287970:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767263.287975:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767263.287979:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099034624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767263.287988:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099034752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767263.288015:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767263.288029:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767263/real 1775767263] req@ffff8b2382daca80 x1862022099034752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767263.288044:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099034752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767263.288051:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767263.288053:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767263.288055:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099034752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767263.288082:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099034880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767263.288098:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767263.288105:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767263/real 1775767263] req@ffff8b2382daf100 x1862022099034880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767263.288114:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022099034880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767263.288124:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767263.288126:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767263.288128:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099034880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767263.288131:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099035008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767263.288142:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767263.288148:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767263/real 1775767263] req@ffff8b2382dac000 x1862022099035008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767263.288155:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022099035008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767263.288160:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767263.288161:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767263.288163:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099035008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767263.288166:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099035136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767263.288176:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767263.288181:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767263/real 1775767263] req@ffff8b2382dac380 x1862022099035136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767263.288188:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099035136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767263.288193:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767263.288195:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767263.288198:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099035136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767264.183653:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767265.207643:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767266.231745:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767267.255653:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767268.279720:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767268.407566:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767268.407581:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767268.407587:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767268.407601:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4729 00000100:00080000:3.0:1775767268.407617:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767268.407625:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767268.407674:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099035264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767268.407710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767268.407715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767268.407717:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767268.407722:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4729 00000100:00080000:3.0:1775767268.407726:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767268.407732:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767268.407744:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767268.407752:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099035264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767268.407760:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099035392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767268.407774:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767268.407778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767268.407780:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767268.407798:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4729 00000100:00080000:3.0:1775767268.407803:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767268.407809:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767268.407815:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767268.407829:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099035520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767268.407839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767268.407843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767268.407845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767268.407850:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4729 00000100:00080000:3.0:1775767268.407854:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775767268.407859:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767268.407860:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767268.407866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767268.407881:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099035648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767268.407892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767268.407896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767268.407898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767268.407903:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4729 00000100:00080000:3.0:1775767268.407907:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767268.407911:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767268/real 1775767268] req@ffff8b23864f3480 x1862022099035264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767268.407914:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767268.407924:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767268.407934:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099035264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767268.407941:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099035776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767268.407946:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767268.407955:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767268.407959:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099035264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767268.407984:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099035392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767268.408002:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767268.408010:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767268/real 1775767268] req@ffff8b23864f2a00 x1862022099035392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767268.408019:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099035392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767268.408024:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767268.408027:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767268.408029:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099035392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767268.408033:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099035520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767268.408044:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767268.408050:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767268/real 1775767268] req@ffff8b23864f0000 x1862022099035520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767268.408057:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099035520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767268.408063:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767268.408064:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767268.408066:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099035520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767268.408070:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099035648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767268.408086:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767268.408092:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767268/real 1775767268] req@ffff8b23864f1180 x1862022099035648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767268.408099:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099035648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767268.408104:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767268.408106:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767268.408107:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099035648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767268.408110:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099035776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767268.408121:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767268.408126:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767268/real 1775767268] req@ffff8b23864f3b80 x1862022099035776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767268.408133:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099035776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767268.408138:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767268.408140:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767268.408141:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099035776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767269.303638:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767270.327670:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767271.351632:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767272.375672:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767273.399694:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767273.527516:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767273.527527:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767273.527533:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767273.527544:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4734 00000100:00080000:3.0:1775767273.527557:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767273.527564:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767273.527608:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099035904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767273.527637:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767273.527641:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767273.527643:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767273.527647:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4734 00000100:00080000:3.0:1775767273.527651:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767273.527655:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767273.527667:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767273.527682:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099036032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767273.527690:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099035904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767273.527694:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767273.527698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767273.527699:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767273.527704:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4734 00000100:00080000:3.0:1775767273.527707:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767273.527712:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767273.527716:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767273.527726:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099036160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767273.527752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767273.527756:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767273.527758:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767273.527762:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4734 00000100:00080000:3.0:1775767273.527765:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767273.527769:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767273.527774:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767273.527784:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099036288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767273.527788:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767273.527793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767273.527797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767273.527798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767273.527802:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4734 00000100:00080000:3.0:1775767273.527805:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767273.527808:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767273.527812:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767273.527821:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099036416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767273.527837:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767273/real 1775767273] req@ffff8b23864f2d80 x1862022099035904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767273.527857:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099035904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767273.527868:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767273.527873:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767273.527877:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099035904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767273.527900:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099036032:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767273.527920:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767273.527930:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767273/real 1775767273] req@ffff8b23864f3100 x1862022099036032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767273.527939:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099036032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767273.527945:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767273.527947:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767273.527949:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099036032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767273.527952:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099036160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767273.527963:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767273.527968:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767273/real 1775767273] req@ffff8b23864f0700 x1862022099036160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767273.527976:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099036160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767273.527986:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767273.527988:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767273.527989:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099036160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767273.527993:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099036288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767273.528005:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767273.528010:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767273/real 1775767273] req@ffff8b23864f2680 x1862022099036288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767273.528017:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099036288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767273.528021:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767273.528023:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767273.528025:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099036288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767273.528028:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099036416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767273.528039:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767273.528044:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767273/real 1775767273] req@ffff8b23864f0380 x1862022099036416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767273.528059:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099036416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767273.528065:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767273.528067:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767273.528070:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099036416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767274.423655:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767275.447661:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767276.471685:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767277.495615:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767278.519626:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767278.647553:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767278.647571:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767278.647576:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767278.647587:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4740 00000100:00080000:2.0:1775767278.647600:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767278.647607:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767278.647653:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099036544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767278.647678:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767278.647681:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767278.647682:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767278.647685:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4740 00000100:00080000:2.0:1775767278.647714:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767278.647719:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767278.647729:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767278.647740:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099036672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767278.647748:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099036544:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767278.647752:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767278.647763:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767278.647767:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767278.647772:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4740 00000100:00080000:2.0:1775767278.647774:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767278.647778:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767278.647783:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767278.647791:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099036800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767278.647799:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767278.647802:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767278.647803:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767278.647811:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4740 00000100:00080000:2.0:1775767278.647813:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767278.647815:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767278.647818:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767278.647828:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099036928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767278.647834:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767278.647837:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767278.647840:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775767278.647841:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775767278.647845:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4740 00000100:00080000:2.0:1775767278.647847:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767278.647850:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767278.647854:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767278.647862:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099037056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767278.647902:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767278/real 1775767278] req@ffff8b23b9ea4700 x1862022099036544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767278.647924:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099036544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767278.647936:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767278.647941:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767278.647945:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099036544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767278.647972:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099036672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767278.647997:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767278.648012:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767278/real 1775767278] req@ffff8b23b9ea5880 x1862022099036672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767278.648022:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099036672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767278.648027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767278.648030:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767278.648032:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099036672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767278.648037:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099036800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767278.648053:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767278.648060:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767278/real 1775767278] req@ffff8b23b9ea4000 x1862022099036800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767278.648067:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099036800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767278.648073:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767278.648075:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767278.648077:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099036800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767278.648080:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099036928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767278.648092:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767278.648098:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767278/real 1775767278] req@ffff8b23b9ea6a00 x1862022099036928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767278.648105:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099036928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767278.648110:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767278.648112:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767278.648114:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099036928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767278.648117:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099037056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767278.648127:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767278.648133:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767278/real 1775767278] req@ffff8b23b9ea4a80 x1862022099037056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767278.648145:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099037056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767278.648150:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767278.648152:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767278.648153:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099037056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767279.543591:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767280.567629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767281.591749:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767282.615677:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767283.639771:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767283.767581:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767283.767592:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767283.767597:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767283.767607:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4745 00000100:00080000:3.0:1775767283.767621:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767283.767628:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767283.767701:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099037184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767283.767729:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767283.767732:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767283.767733:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767283.767736:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4745 00000100:00080000:3.0:1775767283.767739:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767283.767742:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767283.767752:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767283.767764:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099037312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767283.767773:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767283.767775:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767283.767776:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767283.767780:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4745 00000100:00080000:3.0:1775767283.767782:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767283.767785:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767283.767788:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767283.767802:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099037440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767283.767805:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099037184:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767283.767810:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767283.767815:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767283.767816:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767283.767820:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4745 00000100:00080000:3.0:1775767283.767822:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767283.767824:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767283.767827:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767283.767837:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099037568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767283.767844:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767283.767846:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767283.767847:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767283.767850:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4745 00000100:00080000:3.0:1775767283.767851:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767283.767854:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767283.767856:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767283.767863:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099037696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767283.767896:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767283.767951:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767283/real 1775767283] req@ffff8b23864f0a80 x1862022099037184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767283.767974:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099037184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767283.767986:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767283.767991:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767283.767995:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099037184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767283.768004:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099037312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767283.768030:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767283.768044:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767283/real 1775767283] req@ffff8b23864f1f80 x1862022099037312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767283.768067:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099037312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767283.768076:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767283.768079:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767283.768083:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099037312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767283.768114:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099037440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767283.768137:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767283.768149:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767283/real 1775767283] req@ffff8b23864f1c00 x1862022099037440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767283.768161:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099037440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767283.768170:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767283.768175:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767283.768179:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099037440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767283.768186:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099037568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767283.768209:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767283.768220:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767283/real 1775767283] req@ffff8b23864f0380 x1862022099037568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767283.768233:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099037568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767283.768249:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767283.768252:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767283.768256:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099037568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767283.768263:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099037696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767283.768286:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767283.768297:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767283/real 1775767283] req@ffff8b23864f2680 x1862022099037696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767283.768311:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099037696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767283.768320:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767283.768324:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767283.768327:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099037696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767284.663623:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767285.687732:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767286.711689:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767287.735695:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767288.759693:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767288.887569:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767288.887578:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767288.887583:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767288.887614:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4750 00000100:00080000:3.0:1775767288.887626:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767288.887633:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767288.887676:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099037824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767288.887703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767288.887705:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767288.887707:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767288.887711:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4750 00000100:00080000:3.0:1775767288.887715:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767288.887719:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767288.887729:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767288.887746:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099037952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767288.887758:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099037824:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767288.887762:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767288.887767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767288.887769:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767288.887773:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4750 00000100:00080000:3.0:1775767288.887776:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767288.887779:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767288.887784:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767288.887795:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099038080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767288.887802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767288.887804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767288.887805:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767288.887807:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4750 00000100:00080000:3.0:1775767288.887811:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767288.887813:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767288.887816:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767288.887825:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767288.887826:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099038208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767288.887834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767288.887837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767288.887838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767288.887841:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4750 00000100:00080000:3.0:1775767288.887844:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767288.887846:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767288.887849:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767288.887855:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099038336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767288.887876:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767288/real 1775767288] req@ffff8b23864f0700 x1862022099037824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767288.887896:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099037824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767288.887908:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767288.887912:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767288.887915:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099037824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767288.887934:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099037952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767288.887953:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767288.887963:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767288/real 1775767288] req@ffff8b23864f3100 x1862022099037952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767288.887976:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099037952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767288.887985:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767288.887988:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767288.887992:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099037952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767288.887998:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099038080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767288.888020:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767288.888032:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767288/real 1775767288] req@ffff8b23864f2d80 x1862022099038080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767288.888044:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099038080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767288.888052:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767288.888055:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767288.888059:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099038080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767288.888065:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099038208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767288.888087:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767288.888099:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767288/real 1775767288] req@ffff8b23864f3b80 x1862022099038208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767288.888112:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099038208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767288.888118:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767288.888120:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767288.888122:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099038208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767288.888127:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099038336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767288.888142:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767288.888149:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767288/real 1775767288] req@ffff8b23864f1180 x1862022099038336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767288.888157:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099038336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767288.888162:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767288.888164:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767288.888166:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099038336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767289.783619:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767290.807701:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767291.831667:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767292.855633:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767293.879615:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767294.007572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767294.007585:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767294.007592:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767294.007605:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4755 00000100:00080000:3.0:1775767294.007622:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767294.007631:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767294.007683:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099038464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767294.007713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767294.007719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767294.007721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767294.007726:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4755 00000100:00080000:3.0:1775767294.007731:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767294.007736:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767294.007747:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767294.007764:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099038592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767294.007770:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099038464:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767294.007777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767294.007781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767294.007783:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767294.007787:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4755 00000100:00080000:3.0:1775767294.007791:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767294.007794:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767294.007798:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767294.007808:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099038720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767294.007817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767294.007820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767294.007821:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767294.007825:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4755 00000100:00080000:3.0:1775767294.007828:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767294.007831:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767294.007836:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767294.007845:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767294.007846:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099038848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767294.007854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767294.007857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767294.007859:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767294.007862:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4755 00000100:00080000:3.0:1775767294.007865:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767294.007869:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767294.007873:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767294.007882:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099038976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767294.007903:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767294/real 1775767294] req@ffff8b23864f0000 x1862022099038464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767294.007924:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099038464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767294.007935:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767294.007939:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767294.007943:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099038464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767294.007966:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099038592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767294.007985:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767294.007995:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767294/real 1775767294] req@ffff8b23864f2a00 x1862022099038592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767294.008003:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099038592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767294.008009:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767294.008011:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767294.008014:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099038592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767294.008017:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099038720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767294.008028:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767294.008034:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767294/real 1775767294] req@ffff8b23864f3480 x1862022099038720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767294.008042:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099038720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767294.008052:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767294.008055:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767294.008057:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099038720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767294.008060:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099038848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767294.008072:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767294.008077:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767294/real 1775767294] req@ffff8b23864f1180 x1862022099038848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767294.008084:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099038848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767294.008089:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767294.008091:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767294.008093:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099038848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767294.008096:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099038976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767294.008105:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767294.008110:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767294/real 1775767294] req@ffff8b23864f3b80 x1862022099038976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767294.008116:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099038976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767294.008122:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767294.008123:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767294.008126:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099038976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767294.903677:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767295.927631:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767296.951641:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767297.975639:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767298.999696:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767299.127606:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767299.127617:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767299.127623:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767299.127634:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4760 00000100:00080000:2.0:1775767299.127645:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767299.127652:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767299.127695:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099039104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767299.127722:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767299.127725:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767299.127727:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767299.127730:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4760 00000100:00080000:2.0:1775767299.127732:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767299.127735:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767299.127749:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767299.127758:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099039232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767299.127768:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767299.127771:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767299.127773:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767299.127777:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4760 00000100:00080000:2.0:1775767299.127779:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767299.127782:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767299.127785:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767299.127795:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099039360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767299.127796:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099039104:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767299.127814:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767299.127816:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767299.127817:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767299.127820:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4760 00000100:00080000:2.0:1775767299.127822:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767299.127824:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767299.127827:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767299.127835:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022099039488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767299.127841:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767299.127843:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767299.127844:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767299.127846:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4760 00000100:00080000:2.0:1775767299.127848:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767299.127851:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767299.127854:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767299.127861:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099039616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767299.127892:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767299.127952:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767299/real 1775767299] req@ffff8b23b9ea6d80 x1862022099039104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767299.127975:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099039104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767299.127986:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767299.127991:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767299.127996:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099039104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767299.128004:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099039232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767299.128033:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767299.128049:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767299/real 1775767299] req@ffff8b23b9ea6680 x1862022099039232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767299.128063:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099039232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767299.128072:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767299.128075:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767299.128079:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099039232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767299.128108:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099039360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767299.128128:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767299.128138:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767299/real 1775767299] req@ffff8b23b9ea5f80 x1862022099039360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767299.128150:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099039360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767299.128158:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767299.128161:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767299.128163:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099039360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767299.128167:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099039488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767299.128183:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767299.128189:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767299/real 1775767299] req@ffff8b23b9ea5500 x1862022099039488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767299.128197:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022099039488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767299.128202:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767299.128204:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767299.128206:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099039488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767299.128209:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099039616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767299.128221:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767299.128226:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767299/real 1775767299] req@ffff8b23b9ea5c00 x1862022099039616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767299.128238:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099039616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767299.128244:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767299.128245:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767299.128247:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099039616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767300.023661:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767301.047705:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767302.071611:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767303.095653:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767304.119626:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767304.247636:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767304.247651:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767304.247669:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767304.247682:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4765 00000100:00080000:3.0:1775767304.247698:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767304.247706:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767304.247754:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099039744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767304.247786:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767304.247790:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767304.247791:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767304.247794:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4765 00000100:00080000:3.0:1775767304.247798:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767304.247801:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767304.247810:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767304.247822:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099039872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767304.247832:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767304.247835:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767304.247837:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767304.247839:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4765 00000100:00080000:3.0:1775767304.247841:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767304.247844:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767304.247848:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767304.247850:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099039744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767304.247858:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099040000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767304.247869:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767304.247872:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767304.247874:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767304.247879:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4765 00000100:00080000:3.0:1775767304.247883:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767304.247887:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767304.247893:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767304.247917:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099040128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767304.247937:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767304.247937:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767304.247943:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767304.247945:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767304.247951:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4765 00000100:00080000:3.0:1775767304.247956:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767304.247960:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767304.247967:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767304.247980:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099040256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767304.247998:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767304/real 1775767304] req@ffff8b23864f2d80 x1862022099039744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767304.248023:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099039744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767304.248036:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767304.248042:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767304.248046:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099039744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767304.248061:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099039872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767304.248089:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767304.248104:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767304/real 1775767304] req@ffff8b23864f3100 x1862022099039872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767304.248126:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099039872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767304.248135:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767304.248139:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767304.248142:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099039872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767304.248172:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099040000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767304.248195:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767304.248207:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767304/real 1775767304] req@ffff8b23864f0700 x1862022099040000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767304.248220:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099040000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767304.248229:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767304.248232:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767304.248236:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099040000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767304.248242:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099040128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767304.248265:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767304.248275:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767304/real 1775767304] req@ffff8b23864f2680 x1862022099040128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767304.248288:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099040128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767304.248304:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767304.248307:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767304.248310:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099040128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767304.248316:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099040256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767304.248333:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767304.248340:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767304/real 1775767304] req@ffff8b23864f0380 x1862022099040256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767304.248347:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099040256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767304.248353:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767304.248356:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767304.248358:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099040256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767305.143629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767306.167585:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767307.191666:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767308.216897:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767309.239674:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767309.367549:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767309.367558:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767309.367563:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767309.367604:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4770 00000100:00080000:3.0:1775767309.367617:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767309.367729:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767309.367772:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099040384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767309.367812:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767309.367822:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767309.367825:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767309.367831:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4770 00000100:00080000:3.0:1775767309.367841:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767309.367847:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767309.367857:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767309.367874:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099040512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767309.367888:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767309.367892:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767309.367894:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767309.367898:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4770 00000100:00080000:3.0:1775767309.367902:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767309.367904:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099040384:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767309.367910:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767309.367916:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767309.367931:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099040640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767309.367942:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767309.367945:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767309.367947:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767309.367952:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4770 00000100:00080000:3.0:1775767309.367956:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767309.367960:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767309.367965:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767309.367977:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099040768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767309.367985:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767309.367986:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767309.367989:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767309.367991:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767309.367996:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4770 00000100:00080000:3.0:1775767309.368000:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767309.368003:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767309.368007:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767309.368015:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099040896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767309.368027:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767309/real 1775767309] req@ffff8b23864f1c00 x1862022099040384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767309.368042:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099040384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767309.368051:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767309.368055:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767309.368059:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099040384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767309.368079:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099040512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767309.368095:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767309.368103:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767309/real 1775767309] req@ffff8b23864f1f80 x1862022099040512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767309.368111:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099040512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767309.368117:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767309.368119:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767309.368121:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099040512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767309.368125:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099040640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767309.368134:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767309.368140:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767309/real 1775767309] req@ffff8b23864f0a80 x1862022099040640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767309.368147:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099040640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767309.368153:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767309.368154:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767309.368156:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099040640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767309.368159:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099040768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767309.368172:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767309.368177:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767309/real 1775767309] req@ffff8b23864f0380 x1862022099040768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767309.368185:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099040768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767309.368189:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767309.368191:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767309.368193:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099040768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767309.368196:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099040896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767309.368205:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767309.368210:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767309/real 1775767309] req@ffff8b23864f2680 x1862022099040896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767309.368217:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099040896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767309.368222:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767309.368224:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767309.368225:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099040896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767310.263625:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767311.287635:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767312.311642:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767313.335641:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767314.359588:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767314.487505:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767314.487513:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767314.487517:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767314.487535:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4775 00000100:00080000:3.0:1775767314.487547:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767314.487554:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767314.487590:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099041024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767314.487612:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767314.487615:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767314.487616:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767314.487619:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4775 00000100:00080000:3.0:1775767314.487622:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767314.487625:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767314.487633:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767314.487643:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099041152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767314.487654:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099041024:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767314.487662:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767314.487665:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767314.487666:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767314.487669:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4775 00000100:00080000:3.0:1775767314.487672:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767314.487674:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767314.487677:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767314.487686:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099041280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767314.487692:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767314.487694:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767314.487695:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767314.487698:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4775 00000100:00080000:3.0:1775767314.487702:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767314.487704:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767314.487707:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767314.487714:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099041408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767314.487720:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767314.487722:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767314.487723:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767314.487726:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4775 00000100:00080000:3.0:1775767314.487731:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767314.487734:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767314.487740:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767314.487752:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099041536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767314.487757:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099041152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767314.487821:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099041280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767314.487849:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099041408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767314.487869:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099041536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767315.383610:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767316.407637:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767317.431628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775767317.559762:0:2364:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775767317.559808:0:2364:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775767317.559826:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767317.559873:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767317.559884:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767317.559894:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767317.559904:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775767317.559930:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767314/real 1775767317] req@ffff8b23864f1180 x1862022099041536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767317.559960:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099041536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767317.559973:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767317.559978:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767317.559982:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099041536:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767317.559991:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767314/real 1775767317] req@ffff8b23864f3100 x1862022099041152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767317.560005:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099041152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767317.560014:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767317.560017:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767317.560020:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099041152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767317.560029:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767314/real 1775767317] req@ffff8b23864f2d80 x1862022099041280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767317.560046:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099041280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767317.560053:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767317.560057:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767317.560060:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099041280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767317.560067:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767314/real 1775767317] req@ffff8b23864f3b80 x1862022099041408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767317.560079:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099041408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767317.560087:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767317.560091:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767317.560094:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099041408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767317.560101:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767314/real 1775767317] req@ffff8b23864f0700 x1862022099041024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767317.560115:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099041024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767317.560123:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767317.560127:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767317.560130:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099041024:192.168.202.128@tcp:250:kworker.0 00010000:00010000:0.0:1775767318.455683:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767319.479620:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767319.607567:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767319.607582:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767319.607588:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767319.607603:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4781 00000100:00080000:0.0:1775767319.607619:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767319.607628:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767319.607692:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099041664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767319.607729:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767319.607733:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767319.607736:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767319.607740:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4781 00000100:00080000:0.0:1775767319.607744:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767319.607750:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767319.607762:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767319.607780:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099041664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767319.607783:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099041792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767319.607802:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767319.607807:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767319.607808:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767319.607813:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4781 00000100:00080000:0.0:1775767319.607817:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767319.607822:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767319.607828:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767319.607842:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099041920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767319.607851:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767319.607855:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767319.607857:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767319.607862:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4781 00000100:00080000:0.0:1775767319.607866:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767319.607870:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767319.607876:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767319.607882:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767319.607891:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099042048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767319.607902:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767319.607906:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767319.607908:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767319.607912:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4781 00000100:00080000:0.0:1775767319.607916:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767319.607920:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767319.607925:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767319.607938:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099042176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767319.607946:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767319/real 1775767319] req@ffff8b2382dae680 x1862022099041664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767319.607972:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099041664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767319.607985:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767319.607990:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767319.607996:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099041664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767319.608032:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099041792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767319.608064:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767319.608078:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767319/real 1775767319] req@ffff8b2382dad500 x1862022099041792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767319.608092:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099041792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767319.608100:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767319.608104:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767319.608108:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099041792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767319.608115:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099041920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767319.608139:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767319.608151:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767319/real 1775767319] req@ffff8b2382daea00 x1862022099041920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767319.608164:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099041920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767319.608173:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767319.608176:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767319.608180:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099041920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767319.608186:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099042048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767319.608688:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767319.608728:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767319/real 1775767319] req@ffff8b2382daed80 x1862022099042048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767319.608754:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099042048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767319.608767:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767319.608772:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767319.608777:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099042048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767319.608791:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099042176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767319.608827:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767319.608840:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767319/real 1775767319] req@ffff8b2382daf480 x1862022099042176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767319.608853:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099042176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767319.608862:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767319.608866:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767319.608869:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099042176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767320.503666:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767321.527625:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767322.551653:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767323.575603:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767324.599636:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767324.727608:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767324.727616:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767324.727620:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767324.727629:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4786 00000100:00080000:3.0:1775767324.727640:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767324.727646:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767324.727680:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099042304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767324.727708:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767324.727712:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767324.727714:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767324.727718:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4786 00000100:00080000:3.0:1775767324.727721:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767324.727727:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767324.727737:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767324.727754:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099042304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767324.727754:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099042432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767324.727775:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767324.727778:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767324.727780:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767324.727785:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4786 00000100:00080000:3.0:1775767324.727790:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767324.727794:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767324.727799:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767324.727809:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099042560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767324.727816:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767324.727818:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767324.727820:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767324.727823:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4786 00000100:00080000:3.0:1775767324.727826:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775767324.727830:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767324.727833:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767324.727838:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767324.727847:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099042688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767324.727853:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767324.727856:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767324.727857:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767324.727860:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4786 00000100:00080000:3.0:1775767324.727863:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767324.727865:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767324.727869:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767324.727878:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099042816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767324.727886:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767324/real 1775767324] req@ffff8b23864f3480 x1862022099042304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767324.727908:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099042304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767324.727920:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767324.727924:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767324.727929:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099042304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767324.727992:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099042432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767324.728018:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767324.728033:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767324/real 1775767324] req@ffff8b23864f2a00 x1862022099042432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767324.728047:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099042432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767324.728055:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767324.728058:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767324.728062:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099042432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767324.728069:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099042560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767324.728090:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767324.728102:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767324/real 1775767324] req@ffff8b23864f0000 x1862022099042560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767324.728121:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099042560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767324.728129:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767324.728133:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767324.728136:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099042560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767324.728142:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099042688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767324.728166:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767324.728177:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767324/real 1775767324] req@ffff8b23864f0700 x1862022099042688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767324.728190:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099042688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767324.728198:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767324.728201:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767324.728205:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099042688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767324.728210:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099042816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767324.728227:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767324.728235:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767324/real 1775767324] req@ffff8b23864f3b80 x1862022099042816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767324.728246:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099042816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767324.728367:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767324.728372:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767324.728375:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099042816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767325.623649:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767326.647668:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767327.671636:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767328.695625:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767329.719615:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767329.847626:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767329.847641:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767329.847647:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767329.847660:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4791 00000100:00080000:0.0:1775767329.847675:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767329.847682:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767329.847736:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022099042944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767329.847777:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767329.847782:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767329.847784:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767329.847788:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4791 00000100:00080000:0.0:1775767329.847792:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767329.847797:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767329.847810:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767329.847826:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022099043072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767329.847843:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775767329.847845:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099042944:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767329.847853:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767329.847857:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767329.847868:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4791 00000100:00080000:0.0:1775767329.847874:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767329.847879:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767329.847885:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767329.847905:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022099043200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767329.847916:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767329.847921:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767329.847924:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767329.847929:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4791 00000100:00080000:0.0:1775767329.847934:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767329.847941:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775767329.847942:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:0.0:1775767329.847956:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767329.847974:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022099043328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767329.847987:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767329.847991:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767329.847993:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767329.847999:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4791 00000100:00080000:0.0:1775767329.848003:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767329.848006:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767329/real 1775767329] req@ffff8b2382dafb80 x1862022099042944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767329.848012:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767329.848022:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767329.848050:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022099042944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767329.848052:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099043456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767329.848119:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767329.848126:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767329.848133:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099042944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767329.848166:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099043072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767329.848239:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767329.848256:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767329/real 1775767329] req@ffff8b2382dadf80 x1862022099043072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767329.848271:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022099043072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767329.848280:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767329.848284:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767329.848289:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099043072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767329.848297:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099043200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767329.848323:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767329.848336:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767329/real 1775767329] req@ffff8b2382dace00 x1862022099043200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767329.848349:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022099043200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767329.848369:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767329.848373:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767329.848376:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099043200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767329.848384:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099043328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767329.848407:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767329.848418:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767329/real 1775767329] req@ffff8b2382dac700 x1862022099043328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767329.848432:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022099043328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767329.848440:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767329.848443:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767329.848447:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099043328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767329.848479:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099043456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767329.848502:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767329.848514:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767329/real 1775767329] req@ffff8b2382daf480 x1862022099043456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767329.848527:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099043456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767329.848535:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767329.848539:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767329.848542:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099043456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767330.743694:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767331.767663:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767332.791658:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767333.815612:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767334.839694:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767334.967539:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767334.967547:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767334.967551:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767334.967559:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4796 00000100:00080000:0.0:1775767334.967568:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767334.967574:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767334.967607:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099043584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767334.968934:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767334.968938:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767334.968939:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767334.968943:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4796 00000100:00080000:0.0:1775767334.968948:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767334.968951:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767334.968959:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767334.968985:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099043712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767334.968995:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767334.968999:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767334.969002:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767334.969004:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099043584:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767334.969007:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4796 00000100:00080000:0.0:1775767334.969012:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767334.969016:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767334.969020:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767334.969030:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099043840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767334.969036:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767334.969039:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767334.969040:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767334.969043:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4796 00000100:00080000:0.0:1775767334.969045:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767334.969047:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767334.969051:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767334.969057:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099043968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767334.969064:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767334.969066:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767334.969067:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767334.969069:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4796 00000100:00080000:0.0:1775767334.969072:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767334.969078:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775767334.969078:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:0.0:1775767334.969085:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767334.969092:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099044096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767334.969141:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767334/real 1775767334] req@ffff8b2382daed80 x1862022099043584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767334.969177:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099043584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767334.969188:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767334.969193:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767334.969198:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099043584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767334.969224:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099043712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767334.969251:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767334.969265:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767334/real 1775767334] req@ffff8b2382daea00 x1862022099043712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767334.969278:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099043712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767334.969303:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767334.969307:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767334.969311:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099043712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767334.969317:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099043840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767334.969341:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767334.969353:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767334/real 1775767334] req@ffff8b2382dad500 x1862022099043840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767334.969365:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099043840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767334.969374:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767334.969377:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767334.969380:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099043840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767334.969386:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099043968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767334.969405:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767334.969416:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767334/real 1775767334] req@ffff8b2382dae680 x1862022099043968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767334.969428:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099043968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767334.969436:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767334.969439:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767334.969443:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099043968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767334.969449:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099044096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767334.969507:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767334.969519:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767334/real 1775767334] req@ffff8b2382dac380 x1862022099044096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767334.969532:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099044096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767334.969540:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767334.969543:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767334.969547:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099044096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767335.863645:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767336.887662:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767337.911680:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767338.935609:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767339.959697:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767340.087559:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767340.087567:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767340.087571:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767340.087578:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4801 00000100:00080000:1.0:1775767340.087587:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767340.087628:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767340.087661:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099044224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767340.087690:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767340.087693:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767340.087694:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767340.087697:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4801 00000100:00080000:1.0:1775767340.087699:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767340.087702:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767340.087709:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767340.087719:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099044352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767340.087728:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767340.087730:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767340.087732:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767340.087737:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099044224:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767340.087739:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4801 00000100:00080000:1.0:1775767340.087744:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767340.087747:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767340.087751:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767340.087759:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099044480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767340.087765:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767340.087768:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767340.087769:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767340.087772:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4801 00000100:00080000:1.0:1775767340.087776:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767340.087778:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767340.087780:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767340.087788:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099044608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767340.087796:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767340.087800:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767340.087802:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775767340.087806:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775767340.087810:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4801 00000100:00080000:1.0:1775767340.087817:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767340.087822:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767340.087828:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767340.087841:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099044736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767340.087851:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767340/real 1775767340] req@ffff8b23864fed80 x1862022099044224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767340.087870:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099044224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767340.087878:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767340.087881:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767340.087883:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099044224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767340.087903:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099044352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767340.087919:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767340.087927:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767340/real 1775767340] req@ffff8b23864ff800 x1862022099044352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767340.087935:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099044352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767340.087940:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767340.087942:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767340.087945:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099044352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767340.087948:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099044480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767340.087959:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767340.087964:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767340/real 1775767340] req@ffff8b23864fdf80 x1862022099044480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767340.087971:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099044480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767340.087976:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767340.087978:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767340.087980:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099044480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767340.087983:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099044608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767340.087993:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767340.087998:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767340/real 1775767340] req@ffff8b23864fc380 x1862022099044608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767340.088005:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099044608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767340.088011:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767340.088020:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767340.088022:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099044608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767340.088026:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099044736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767340.088038:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767340.088043:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767340/real 1775767340] req@ffff8b23864fc000 x1862022099044736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767340.088050:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099044736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767340.088055:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767340.088057:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767340.088059:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099044736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767340.983578:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767342.007663:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767343.031593:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767344.055805:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767345.079644:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767345.207561:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767345.207572:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767345.207576:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767345.207584:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4806 00000100:00080000:3.0:1775767345.207594:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767345.207599:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767345.207634:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099044864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767345.207658:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767345.207661:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767345.207663:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767345.207665:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4806 00000100:00080000:3.0:1775767345.207668:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767345.207670:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767345.207682:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767345.207691:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099044992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767345.207700:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767345.207702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767345.207704:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767345.207717:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099044864:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767345.207745:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4806 00000100:00080000:3.0:1775767345.207749:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767345.207754:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767345.207759:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767345.207773:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099045120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767345.207781:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767345.207791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767345.207795:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767345.207798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767345.207804:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4806 00000100:00080000:3.0:1775767345.207808:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767345.207813:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767345.207819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767345.207831:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099045248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767345.207835:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767345/real 1775767345] req@ffff8b23864f2d80 x1862022099044864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767345.207850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767345.207857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767345.207859:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767345.207866:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099044864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767345.207878:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767345.207883:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00080000:3.0:1775767345.207887:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4806 00000100:00100000:0.0:1775767345.207892:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099044864:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767345.207893:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767345.207901:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767345.207906:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099044992:192.168.202.128@tcp:38:kworker.0 00000008:00100000:3.0:1775767345.207908:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767345.207928:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099045376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767345.207945:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767345.209320:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767345/real 1775767345] req@ffff8b23864f3100 x1862022099044992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767345.209337:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099044992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767345.209352:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767345.209356:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767345.209360:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099044992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767345.209384:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099045120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767345.209408:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767345.209419:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767345/real 1775767345] req@ffff8b23864f1180 x1862022099045120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767345.209431:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099045120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767345.209440:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767345.209443:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767345.209446:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099045120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767345.209474:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099045248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767345.209511:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767345.209522:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767345/real 1775767345] req@ffff8b23864f2680 x1862022099045248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767345.209534:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099045248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767345.209542:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767345.209545:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767345.209549:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099045248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767345.209555:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099045376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767345.209572:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767345.209581:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767345/real 1775767345] req@ffff8b23864f0380 x1862022099045376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767345.209593:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099045376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767345.209601:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767345.209604:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767345.209607:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099045376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767346.103694:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767347.127609:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767348.151697:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767349.175666:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767350.199707:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767350.327548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767350.327556:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767350.327560:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767350.327569:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4811 00000100:00080000:3.0:1775767350.327578:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767350.327583:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767350.327616:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099045504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767350.327639:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767350.327641:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767350.327643:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767350.327645:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4811 00000100:00080000:3.0:1775767350.327648:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767350.327651:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767350.327658:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767350.327667:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099045632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767350.327675:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767350.327678:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767350.327680:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767350.327692:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4811 00000100:00080000:3.0:1775767350.327695:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767350.327702:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767350.327707:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767350.327721:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099045504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767350.327722:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099045760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767350.327746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767350.327751:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767350.327752:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767350.327757:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4811 00000100:00080000:3.0:1775767350.327761:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767350.327764:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767350.327768:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767350.327779:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099045888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767350.327790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767350.327792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767350.327794:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767350.327797:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4811 00000100:00080000:3.0:1775767350.327800:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767350.327802:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767350.327805:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767350.327813:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767350.327823:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099046016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767350.327870:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767350/real 1775767350] req@ffff8b23864f0a80 x1862022099045504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767350.327887:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099045504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767350.327897:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767350.327901:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767350.327904:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099045504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767350.327911:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099045632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767350.327928:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767350.327937:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767350/real 1775767350] req@ffff8b23864f1f80 x1862022099045632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767350.327946:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099045632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767350.327951:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767350.327953:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767350.327955:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099045632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767350.327987:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099045760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767350.328001:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767350.328008:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767350/real 1775767350] req@ffff8b23864f1c00 x1862022099045760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767350.328015:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099045760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767350.328039:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767350.328042:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767350.328044:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099045760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767350.328049:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099045888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767350.328066:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767350.328073:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767350/real 1775767350] req@ffff8b23864f0380 x1862022099045888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767350.328086:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099045888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767350.328092:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767350.328094:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767350.328096:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099045888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767350.328100:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099046016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767350.328110:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767350.328115:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767350/real 1775767350] req@ffff8b23864f2680 x1862022099046016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767350.328123:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099046016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767350.328128:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767350.328130:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767350.328131:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099046016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767351.223630:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767352.247625:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767353.271636:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767354.295644:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767355.319638:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767355.447580:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767355.447590:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767355.447595:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767355.447605:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4816 00000100:00080000:1.0:1775767355.447618:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767355.447625:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767355.447683:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099046144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767355.447707:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767355.447710:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767355.447711:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767355.447714:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4816 00000100:00080000:1.0:1775767355.447717:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767355.447721:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767355.447730:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767355.447743:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099046272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767355.447758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767355.447761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767355.447763:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767355.447766:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4816 00000100:00080000:1.0:1775767355.447769:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767355.447773:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767355.447777:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767355.447793:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099046400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767355.447796:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099046144:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767355.447822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767355.447824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767355.447826:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767355.447831:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4816 00000100:00080000:1.0:1775767355.447836:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767355.447839:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767355.447843:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767355.447855:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099046528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767355.447861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767355.447864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767355.447866:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767355.447869:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4816 00000100:00080000:1.0:1775767355.447873:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767355.447876:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775767355.447883:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:1.0:1775767355.447883:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767355.447896:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099046656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767355.447929:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767355/real 1775767355] req@ffff8b23864fce00 x1862022099046144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767355.447950:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099046144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767355.447958:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767355.447962:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767355.447965:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099046144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767355.447997:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099046272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767355.448016:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767355.448026:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767355/real 1775767355] req@ffff8b23864fd180 x1862022099046272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767355.448034:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099046272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767355.448040:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767355.448042:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767355.448044:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099046272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767355.448065:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099046400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767355.448077:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767355.448084:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767355/real 1775767355] req@ffff8b23864fea00 x1862022099046400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767355.448091:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099046400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767355.448101:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767355.448104:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767355.448106:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099046400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767355.448110:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099046528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767355.448120:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767355.448126:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767355/real 1775767355] req@ffff8b23864fca80 x1862022099046528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767355.448133:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099046528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767355.448138:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767355.448140:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767355.448142:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099046528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767355.448146:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099046656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767355.448155:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767355.448161:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767355/real 1775767355] req@ffff8b23864fe680 x1862022099046656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767355.448168:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099046656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767355.448173:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767355.448183:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767355.448186:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099046656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767356.343665:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767357.367687:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767358.391606:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767359.415684:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767360.439608:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767360.567593:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767360.567603:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767360.567609:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767360.567620:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4821 00000100:00080000:1.0:1775767360.567634:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767360.567642:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767360.567690:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099046784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767360.567715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767360.567718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767360.567719:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767360.567722:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4821 00000100:00080000:1.0:1775767360.567725:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767360.567727:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767360.567738:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767360.567746:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099046912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767360.567756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767360.567758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767360.567760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767360.567763:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4821 00000100:00080000:1.0:1775767360.567765:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767360.567768:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767360.567771:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767360.567785:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099046784:192.168.202.128@tcp:250:kworker.0 00000800:00000100:0.0:1775767360.567869:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767360.567919:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767360/real 1775767360] req@ffff8b23864fdc00 x1862022099046784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767360.567959:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099046784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767360.567972:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1775767360.567981:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099047040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767360.567982:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767360.567991:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099046784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767360.568010:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099046912:192.168.202.128@tcp:38:kworker.0 00000100:00080000:1.0:1775767360.568021:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767360.568025:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767360.568028:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767360.568036:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4821 00000800:00000100:0.0:1775767360.568045:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775767360.568046:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767360.568058:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767360.568070:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767360.568072:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767360/real 1775767360] req@ffff8b23864fd500 x1862022099046912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767360.568091:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099046912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767360.568097:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099047168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767360.568143:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767360.568146:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767360.568146:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767360.568150:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767360.568163:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4821 00000100:00080000:0.0:1775767360.568165:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00080000:1.0:1775767360.568170:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767360.568176:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099046912:192.168.202.128@tcp:38:kworker.0 00000100:00080000:1.0:1775767360.568176:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767360.568189:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767360.568198:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099047296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767360.568228:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099047040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767360.568259:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767360.568272:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767360/real 1775767360] req@ffff8b23864ff100 x1862022099047040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767360.568286:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099047040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767360.568295:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767360.568299:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767360.568303:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099047040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767360.568310:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099047168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767360.568333:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767360.568344:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767360/real 1775767360] req@ffff8b23864fe300 x1862022099047168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767360.568357:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099047168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767360.568366:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767360.568370:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767360.568374:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099047168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767360.568380:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099047296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767360.568399:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767360.568409:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767360/real 1775767360] req@ffff8b23864ffb80 x1862022099047296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767360.568430:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099047296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767360.568440:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767360.568443:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767360.568447:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099047296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767361.463719:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767362.487621:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767363.511692:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767364.535678:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767365.559696:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767365.687602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767365.687615:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767365.687621:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767365.687634:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4827 00000100:00080000:3.0:1775767365.687650:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767365.687658:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767365.687709:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099047424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767365.687742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767365.687746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767365.687749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767365.687753:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4827 00000100:00080000:3.0:1775767365.687758:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767365.687763:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767365.687776:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767365.687792:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099047424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767365.687796:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099047552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767365.687812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767365.687816:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767365.687818:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767365.687824:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4827 00000100:00080000:3.0:1775767365.687829:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767365.687834:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767365.687840:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767365.687854:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099047680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767365.687865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767365.687869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1775767365.687873:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767365.687873:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767365.687920:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4827 00000100:00080000:3.0:1775767365.687926:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767365.687931:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767365.687938:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767365.687952:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099047808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767365.687963:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767365.687967:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767365/real 1775767365] req@ffff8b23864f1180 x1862022099047424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767365.687969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767365.687973:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767365.687984:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4827 00000100:00080000:3.0:1775767365.687988:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767365.687993:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767365.687996:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099047424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775767365.688001:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775767365.688008:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767365.688014:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767365.688018:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099047936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767365.688020:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099047424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767365.688050:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099047552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767365.688071:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767365.688080:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767365/real 1775767365] req@ffff8b23864f3100 x1862022099047552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767365.688094:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099047552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767365.688100:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767365.688102:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767365.688104:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099047552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767365.688108:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099047680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767365.688119:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767365.688126:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767365/real 1775767365] req@ffff8b23864f2d80 x1862022099047680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767365.688133:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099047680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767365.688138:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767365.688140:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767365.688142:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099047680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767365.688145:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099047808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767365.688157:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767365.688162:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767365/real 1775767365] req@ffff8b23864f3b80 x1862022099047808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767365.688169:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099047808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767365.688179:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767365.688181:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767365.688183:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099047808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767365.688186:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099047936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767365.688196:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767365.688201:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767365/real 1775767365] req@ffff8b23864f0700 x1862022099047936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767365.688207:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099047936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767365.688212:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767365.688214:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767365.688216:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099047936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767366.583658:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767367.607699:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767368.631651:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767369.655666:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767370.679602:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767370.807606:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767370.807617:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767370.807622:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767370.807633:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4832 00000100:00080000:1.0:1775767370.807646:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767370.807653:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767370.807702:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099048064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767370.807736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767370.807739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767370.807740:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767370.807743:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4832 00000100:00080000:1.0:1775767370.807746:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767370.807749:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767370.807760:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767370.807772:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099048192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767370.807781:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767370.807784:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767370.807786:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767370.807793:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099048064:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767370.807793:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4832 00000100:00080000:1.0:1775767370.807795:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767370.807801:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767370.807806:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767370.807815:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099048320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767370.807821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767370.807824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767370.807825:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767370.807841:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4832 00000100:00080000:1.0:1775767370.807845:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767370.807849:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767370.807854:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767370.807872:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099048448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767370.807882:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767370.807887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767370.807889:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767370.807894:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4832 00000100:00080000:1.0:1775767370.807897:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767370.807903:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775767370.807905:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:1.0:1775767370.807909:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767370.807926:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099048576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767370.807954:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767370/real 1775767370] req@ffff8b23864ffb80 x1862022099048064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767370.807971:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099048064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767370.807980:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767370.807985:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767370.807988:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099048064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767370.808011:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099048192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767370.808028:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767370.808037:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767370/real 1775767370] req@ffff8b23864fe300 x1862022099048192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767370.808045:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099048192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767370.808051:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767370.808053:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767370.808056:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099048192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767370.808059:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099048320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767370.808070:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767370.808075:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767370/real 1775767370] req@ffff8b23864ff100 x1862022099048320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767370.808083:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099048320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767370.808088:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767370.808090:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767370.808092:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099048320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767370.808095:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099048448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767370.808107:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767370.808112:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767370/real 1775767370] req@ffff8b23864fd500 x1862022099048448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767370.808120:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099048448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767370.808125:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767370.808127:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767370.808128:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099048448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767370.808132:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099048576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767370.808141:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767370.808146:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767370/real 1775767370] req@ffff8b23864fdc00 x1862022099048576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767370.808152:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099048576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767370.808157:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767370.808159:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767370.808160:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099048576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767371.703679:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767372.727615:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767373.751758:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767374.775653:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767375.799633:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767375.927633:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767375.927644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767375.927649:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767375.927660:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4837 00000100:00080000:1.0:1775767375.927674:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767375.927682:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767375.927726:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099048704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767375.927764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767375.927768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767375.927770:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767375.927774:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4837 00000100:00080000:1.0:1775767375.927815:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767375.927822:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767375.927833:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767375.927849:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099048832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767375.927854:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099048704:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767375.927883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767375.927887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767375.927888:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767375.927895:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4837 00000100:00080000:1.0:1775767375.927899:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767375.927905:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767375.927910:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767375.927924:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099048960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767375.927932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767375.927936:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767375.927938:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767375.927941:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4837 00000100:00080000:1.0:1775767375.927959:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767375.927962:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767375.927966:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767375.927987:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775767375.927998:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099049088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767375.928009:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767375.928013:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767375.928016:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767375.928021:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4837 00000100:00080000:1.0:1775767375.928025:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767375.928028:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767375.928035:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767375.928053:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767375/real 1775767375] req@ffff8b23864fe680 x1862022099048704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767375.928054:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099049216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767375.928172:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099048704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767375.928184:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767375.928189:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767375.928194:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099048704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767375.928237:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099048832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767375.928265:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767375.928281:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767375/real 1775767375] req@ffff8b23864fca80 x1862022099048832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767375.928295:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099048832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767375.928304:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767375.928308:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767375.928312:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099048832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767375.928319:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099048960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767375.928337:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767375.928346:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767375/real 1775767375] req@ffff8b23864fea00 x1862022099048960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767375.928354:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099048960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767375.928365:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767375.928368:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767375.928370:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099048960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767375.928374:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099049088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767375.928387:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767375.928392:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767375/real 1775767375] req@ffff8b23864fd180 x1862022099049088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767375.928400:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099049088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767375.928405:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767375.928406:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767375.928408:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099049088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767375.928412:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099049216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767375.928423:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767375.928432:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767375/real 1775767375] req@ffff8b23864fce00 x1862022099049216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767375.928444:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099049216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767375.928508:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767375.928511:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767375.928515:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099049216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767376.823713:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767377.847625:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767378.871736:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767379.895652:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767380.919652:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767381.047569:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767381.047577:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767381.047580:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767381.047589:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4842 00000100:00080000:2.0:1775767381.047598:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767381.047604:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767381.047635:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099049344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767381.047664:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767381.047667:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767381.047671:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767381.047674:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4842 00000100:00080000:2.0:1775767381.047676:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767381.047679:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767381.047686:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767381.047697:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099049472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767381.047706:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767381.047710:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767381.047711:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767381.047714:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4842 00000100:00080000:2.0:1775767381.047717:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767381.047720:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767381.047723:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767381.047745:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022099049600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767381.047759:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767381.047764:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099049344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767381.047764:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767381.047771:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767381.047788:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4842 00000100:00080000:2.0:1775767381.047795:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767381.047798:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767381.047803:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767381.047815:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099049728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767381.047822:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767381.047825:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767381.047826:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767381.047829:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4842 00000100:00080000:2.0:1775767381.047831:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767381.047833:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767381.047837:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767381.047845:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099049856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767381.047907:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099049472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767381.047965:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099049600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767381.047985:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099049728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767381.048007:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099049856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767381.943590:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767382.967698:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767383.991685:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775767384.119874:0:2365:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775767384.119940:0:2365:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775767384.119967:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767384.120019:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767384.120030:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767384.120038:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767384.120049:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775767384.120094:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767381/real 1775767384] req@ffff8b23b9ea5500 x1862022099049600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767384.120129:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022099049600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767384.120148:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767384.120156:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767384.120163:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099049600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767384.120174:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767381/real 1775767384] req@ffff8b23b9ea5f80 x1862022099049728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767384.120189:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099049728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767384.120205:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767384.120209:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767384.120213:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099049728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767384.120224:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767381/real 1775767384] req@ffff8b23b9ea6680 x1862022099049856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767384.120237:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099049856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767384.120245:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767384.120249:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767384.120252:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099049856:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767384.120260:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767381/real 1775767384] req@ffff8b23b9ea7b80 x1862022099049344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767384.120272:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099049344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767384.120279:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767384.120283:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767384.120286:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099049344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767384.120294:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767381/real 1775767384] req@ffff8b23b9ea5c00 x1862022099049472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767384.120308:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099049472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767384.120317:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767384.120320:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767384.120324:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099049472:192.168.202.128@tcp:38:kworker.0 00010000:00010000:1.0:1775767385.015766:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767386.039609:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767386.167673:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767386.167709:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767386.167713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767386.167725:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4847 00000100:00080000:2.0:1775767386.167738:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767386.167744:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767386.167792:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099049984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767386.167843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767386.167848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767386.167851:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767386.167856:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4847 00000100:00080000:2.0:1775767386.167860:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767386.167864:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767386.167877:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767386.167891:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099050112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767386.167904:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767386.167908:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767386.167910:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767386.167915:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4847 00000100:00080000:2.0:1775767386.167919:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767386.167923:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767386.167929:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767386.167940:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099050240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767386.167950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767386.167954:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767386.167956:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767386.167961:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4847 00000100:00080000:2.0:1775767386.167965:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767386.167970:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767386.167976:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767386.167989:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099050368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767386.168001:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767386.168003:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099049984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767386.168008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767386.168012:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767386.168018:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4847 00000100:00080000:2.0:1775767386.168023:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767386.168028:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767386.168036:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767386.168059:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099050496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767386.168120:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767386.168186:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767386/real 1775767386] req@ffff8b23b9ea6d80 x1862022099049984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767386.168213:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099049984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767386.168227:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767386.168232:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767386.168236:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099049984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767386.168245:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099050112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767386.168273:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767386.168289:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767386/real 1775767386] req@ffff8b23b9ea4a80 x1862022099050112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767386.168301:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099050112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767386.168310:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767386.168313:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767386.168318:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099050112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767386.168325:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099050240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767386.168346:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767386.168356:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767386/real 1775767386] req@ffff8b23b9ea6a00 x1862022099050240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767386.168368:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099050240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767386.168377:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767386.168379:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767386.168383:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099050240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767386.168420:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099050368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767386.168442:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767386.168487:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767386/real 1775767386] req@ffff8b23b9ea4000 x1862022099050368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767386.168502:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099050368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767386.168511:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767386.168514:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767386.168518:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099050368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767386.168525:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099050496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767386.168546:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767386.168558:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767386/real 1775767386] req@ffff8b23b9ea5880 x1862022099050496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767386.168570:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099050496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767386.168580:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767386.168583:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767386.168586:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099050496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767387.063631:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767388.087633:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767389.111719:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767390.135640:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767391.159695:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767391.287605:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767391.287613:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767391.287617:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767391.287625:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4852 00000100:00080000:1.0:1775767391.287659:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767391.287666:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767391.287701:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099050624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767391.287726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767391.287729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767391.287731:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767391.287734:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4852 00000100:00080000:1.0:1775767391.287737:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767391.287740:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767391.287748:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767391.287759:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099050752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767391.287768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767391.287771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767391.287773:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767391.287777:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099050624:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767391.287779:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4852 00000100:00080000:1.0:1775767391.287788:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767391.287791:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767391.287795:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767391.287802:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099050880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767391.287809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767391.287812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767391.287813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767391.287815:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4852 00000100:00080000:1.0:1775767391.287817:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767391.287819:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767391.287823:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767391.287830:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099051008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767391.287837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767391.287839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767391.287840:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775767391.287844:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775767391.287847:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4852 00000100:00080000:1.0:1775767391.287851:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767391.287853:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767391.287857:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767391.287865:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099051136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767391.287886:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767391/real 1775767391] req@ffff8b23864fc000 x1862022099050624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767391.287900:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099050624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767391.287908:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767391.287911:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767391.287914:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099050624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767391.287932:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099050752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767391.287946:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767391.287954:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767391/real 1775767391] req@ffff8b23864fc380 x1862022099050752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767391.287966:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099050752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767391.287971:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767391.287973:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767391.287975:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099050752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767391.287979:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099050880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767391.287988:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767391.287994:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767391/real 1775767391] req@ffff8b23864fdf80 x1862022099050880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767391.288001:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099050880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767391.288006:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767391.288008:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767391.288009:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099050880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767391.288013:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099051008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767391.288023:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767391.288028:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767391/real 1775767391] req@ffff8b23864ff800 x1862022099051008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767391.288034:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099051008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767391.288043:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767391.288046:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767391.288047:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099051008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767391.288051:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099051136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767391.288060:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767391.288064:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767391/real 1775767391] req@ffff8b23864fed80 x1862022099051136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767391.288071:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099051136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767391.288076:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767391.288078:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767391.288080:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099051136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767392.183737:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767393.207661:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767394.231680:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767395.255657:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767396.279690:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767396.407557:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767396.407568:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767396.407574:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767396.407608:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4857 00000100:00080000:1.0:1775767396.407633:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767396.407639:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767396.407696:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099051264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767396.407727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767396.407742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767396.407744:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767396.407749:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4857 00000100:00080000:1.0:1775767396.407753:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767396.407758:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767396.407772:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767396.407789:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099051392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767396.407802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767396.407807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775767396.407808:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099051264:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767396.407809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767396.407829:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4857 00000100:00080000:1.0:1775767396.407833:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767396.407836:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767396.407840:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767396.407856:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099051520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767396.407864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767396.407867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767396.407869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767396.407872:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4857 00000100:00080000:1.0:1775767396.407894:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767396.407898:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767396.407904:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767396.407909:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775767396.407920:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099051648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767396.407938:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767396.407942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767396.407944:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767396.407948:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4857 00000100:00080000:1.0:1775767396.407953:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767396.407957:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767396.407962:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767396.407972:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767396/real 1775767396] req@ffff8b23864fed80 x1862022099051264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767396.407973:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099051776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767396.408018:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099051264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767396.408033:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767396.408037:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767396.408051:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099051264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767396.408077:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099051392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767396.408095:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767396.408105:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767396/real 1775767396] req@ffff8b23864ff800 x1862022099051392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767396.408113:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099051392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767396.408119:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767396.408121:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767396.408123:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099051392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767396.408127:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099051520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767396.408138:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767396.408144:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767396/real 1775767396] req@ffff8b23864fdf80 x1862022099051520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767396.408151:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099051520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767396.408157:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767396.408158:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767396.408160:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099051520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767396.408164:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099051648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767396.408176:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767396.408181:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767396/real 1775767396] req@ffff8b23864fc380 x1862022099051648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767396.408189:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099051648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767396.408194:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767396.408196:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767396.408198:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099051648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767396.408201:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099051776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767396.408211:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767396.408215:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767396/real 1775767396] req@ffff8b23864fc000 x1862022099051776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767396.408222:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099051776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767396.408226:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767396.408229:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767396.408231:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099051776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767397.303664:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767398.327697:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767399.351632:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767400.375715:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767401.399732:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767401.527561:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767401.527576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767401.527583:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767401.527596:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4862 00000100:00080000:1.0:1775767401.527611:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767401.527620:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767401.527675:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099051904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767401.527721:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767401.527726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767401.527729:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767401.527737:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4862 00000100:00080000:1.0:1775767401.527742:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767401.527748:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767401.527762:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767401.527777:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099051904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775767401.527784:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099052032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767401.527802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767401.527807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767401.527809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767401.527815:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4862 00000100:00080000:1.0:1775767401.527820:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767401.527826:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767401.527832:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767401.527848:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099052160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767401.527859:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775767401.527865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767401.527870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767401.527873:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767401.527880:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4862 00000100:00080000:1.0:1775767401.527886:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767401.527891:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767401.527898:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767401.527909:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767401/real 1775767401] req@ffff8b23864fce00 x1862022099051904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767401.527919:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099052288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767401.527934:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099051904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767401.527936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767401.527947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767401.527952:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767401.527954:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767401.527959:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4862 00000100:00080000:0.0:1775767401.527960:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767401.527968:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099051904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767401.527969:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767401.527978:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767401.527985:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767401.528001:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099052416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767401.528002:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099052032:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767401.528035:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767401.528053:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767401/real 1775767401] req@ffff8b23864fd180 x1862022099052032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767401.528070:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099052032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767401.528080:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767401.528085:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767401.528089:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099052032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767401.528095:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099052160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767401.528114:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767401.528122:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767401/real 1775767401] req@ffff8b23864fea00 x1862022099052160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767401.528137:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099052160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767401.528144:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767401.528146:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767401.528148:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099052160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767401.528153:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099052288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767401.528171:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767401.528182:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767401/real 1775767401] req@ffff8b23864fca80 x1862022099052288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767401.528210:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099052288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767401.528220:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767401.528224:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767401.528228:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099052288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767401.528245:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099052416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767401.528263:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767401.528272:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767401/real 1775767401] req@ffff8b23864fe680 x1862022099052416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767401.528280:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099052416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767401.528291:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767401.528293:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767401.528296:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099052416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767402.423685:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767403.447735:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767404.471679:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767405.495655:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767406.519627:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767406.647642:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767406.647653:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767406.647665:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767406.647679:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4868 00000100:00080000:2.0:1775767406.647692:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767406.647699:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767406.647746:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099052544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767406.647774:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767406.647778:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767406.647780:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767406.647783:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4868 00000100:00080000:2.0:1775767406.647785:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767406.647788:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767406.647798:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767406.647807:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099052672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767406.647818:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767406.647825:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767406.647828:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767406.647834:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4868 00000100:00080000:2.0:1775767406.647838:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767406.647844:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767406.647849:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099052544:192.168.202.128@tcp:250:kworker.0 00000008:00100000:2.0:1775767406.647853:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767406.647871:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099052800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767406.647883:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767406.647887:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767406.647888:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767406.647892:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4868 00000100:00080000:2.0:1775767406.647897:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767406.647900:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767406.647904:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767406.647914:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099052928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767406.647921:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767406.647924:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767406.647925:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767406.647930:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4868 00000800:00000100:0.0:1775767406.647931:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775767406.647935:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767406.647939:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767406.647943:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767406.647951:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099053056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767406.648019:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767406/real 1775767406] req@ffff8b23b9ea4700 x1862022099052544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767406.648040:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099052544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767406.648049:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767406.648053:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767406.648056:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099052544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767406.648063:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099052672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767406.648103:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767406.648113:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767406/real 1775767406] req@ffff8b23b9ea7480 x1862022099052672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767406.648122:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099052672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767406.648133:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767406.648135:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767406.648138:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099052672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767406.648160:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099052800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767406.648172:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767406.648178:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767406/real 1775767406] req@ffff8b23b9ea6300 x1862022099052800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767406.648186:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099052800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767406.648191:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767406.648193:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767406.648195:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099052800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767406.648199:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099052928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767406.648211:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767406.648216:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767406/real 1775767406] req@ffff8b23b9ea7100 x1862022099052928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767406.648224:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099052928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767406.648229:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767406.648231:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767406.648234:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099052928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767406.648237:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099053056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767406.648248:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767406.648253:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767406/real 1775767406] req@ffff8b23b9ea4380 x1862022099053056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767406.648260:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099053056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767406.648265:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767406.648267:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767406.648269:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099053056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767407.543627:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767408.567670:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767409.591705:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767410.615623:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767411.639629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767411.767631:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767411.767644:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767411.767650:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767411.767664:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4873 00000100:00080000:1.0:1775767411.767678:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767411.767685:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767411.767732:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099053184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767411.767767:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767411.767770:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767411.767773:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767411.767776:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4873 00000100:00080000:1.0:1775767411.767778:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767411.767781:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767411.767791:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767411.767799:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099053312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767411.767808:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767411.767810:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767411.767811:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767411.767814:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4873 00000100:00080000:1.0:1775767411.767816:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767411.767819:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767411.767822:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767411.767833:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099053440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767411.767840:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767411.767842:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767411.767844:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767411.767847:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4873 00000100:00080000:1.0:1775767411.767849:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767411.767851:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767411.767855:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767411.767862:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099053568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767411.767870:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767411.767873:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099053184:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767411.767873:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767411.767874:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767411.767882:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4873 00000100:00080000:1.0:1775767411.767884:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767411.767886:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767411.767889:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767411.767906:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099053696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767411.767951:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767411.767994:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767411/real 1775767411] req@ffff8b23864fdc00 x1862022099053184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767411.768009:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099053184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767411.768018:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767411.768022:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767411.768025:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099053184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767411.768030:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099053312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767411.768046:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767411.768054:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767411/real 1775767411] req@ffff8b23864fd500 x1862022099053312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767411.768062:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099053312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767411.768067:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767411.768069:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767411.768071:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099053312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767411.768075:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099053440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767411.768085:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767411.768091:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767411/real 1775767411] req@ffff8b23864ff100 x1862022099053440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767411.768098:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099053440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767411.768103:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767411.768105:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767411.768107:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099053440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767411.768132:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099053568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767411.768144:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767411.768150:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767411/real 1775767411] req@ffff8b23864fe300 x1862022099053568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767411.768162:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099053568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767411.768167:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767411.768169:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767411.768171:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099053568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767411.768174:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099053696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767411.768184:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767411.768189:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767411/real 1775767411] req@ffff8b23864ffb80 x1862022099053696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767411.768197:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099053696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767411.768202:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767411.768204:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767411.768206:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099053696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767412.663623:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767413.687697:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767414.711618:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767415.735681:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767416.759603:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767416.887616:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767416.887630:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767416.887636:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767416.887648:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4878 00000100:00080000:1.0:1775767416.887662:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767416.887670:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767416.887725:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099053824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767416.887759:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767416.887763:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767416.887765:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767416.887770:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4878 00000100:00080000:1.0:1775767416.887773:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767416.887778:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767416.887790:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767416.887803:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099053952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767416.887816:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767416.887821:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775767416.887825:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099053824:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767416.887834:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767416.887840:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4878 00000100:00080000:1.0:1775767416.887844:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767416.887848:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767416.887853:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767416.887866:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099054080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767416.887877:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767416.887881:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767416.887883:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767416.887887:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4878 00000100:00080000:1.0:1775767416.887908:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767416.887912:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767416.887917:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767416.887923:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775767416.887931:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099054208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767416.887941:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767416.887946:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767416.887948:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767416.887952:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4878 00000100:00080000:1.0:1775767416.887956:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767416.887960:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767416.887964:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767416.887979:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099054336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767416.887984:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767416/real 1775767416] req@ffff8b23864ffb80 x1862022099053824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767416.888014:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099053824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767416.888024:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767416.888029:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767416.888033:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099053824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767416.888056:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099053952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767416.888077:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767416.888086:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767416/real 1775767416] req@ffff8b23864fe300 x1862022099053952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767416.888094:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099053952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767416.888100:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767416.888102:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767416.888104:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099053952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767416.888108:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099054080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767416.888118:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767416.888124:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767416/real 1775767416] req@ffff8b23864ff100 x1862022099054080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767416.888131:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099054080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767416.888141:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767416.888143:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767416.888145:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099054080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767416.888148:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099054208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767416.888159:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767416.888165:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767416/real 1775767416] req@ffff8b23864fd500 x1862022099054208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767416.888171:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099054208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767416.888176:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767416.888178:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767416.888180:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099054208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767416.888183:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099054336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767416.888193:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767416.888198:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767416/real 1775767416] req@ffff8b23864fdc00 x1862022099054336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767416.888204:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099054336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767416.888209:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767416.888210:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767416.888213:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099054336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767417.783622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767418.807683:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767419.831625:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767420.855630:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767421.879670:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767422.007601:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767422.007611:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767422.007616:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767422.007627:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4883 00000100:00080000:3.0:1775767422.007639:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767422.007645:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767422.007688:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099054464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767422.007714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767422.007717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767422.007719:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767422.007721:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4883 00000100:00080000:3.0:1775767422.007724:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767422.007727:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767422.007737:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767422.007747:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099054592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767422.007756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767422.007760:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767422.007761:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767422.007764:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4883 00000100:00080000:3.0:1775767422.007766:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767422.007770:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767422.007774:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767422.007779:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099054464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767422.007789:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099054720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767422.007800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767422.007804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767422.007807:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767422.007811:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4883 00000100:00080000:3.0:1775767422.007815:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767422.007820:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767422.007825:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767422.007840:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099054848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767422.007850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767422.007855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767422.007857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767422.007861:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4883 00000100:00080000:3.0:1775767422.007864:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775767422.007867:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767422.007871:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767422.007876:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767422.007887:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099054976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767422.007919:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767422/real 1775767422] req@ffff8b23864f0000 x1862022099054464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767422.007942:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099054464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767422.007955:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767422.007960:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767422.007964:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099054464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767422.007972:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099054592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767422.008002:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767422.008013:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767422/real 1775767422] req@ffff8b23864f2a00 x1862022099054592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767422.008021:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099054592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767422.008027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767422.008030:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767422.008034:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099054592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767422.008063:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099054720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767422.008086:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767422.008098:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767422/real 1775767422] req@ffff8b23864f3480 x1862022099054720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767422.008107:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099054720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767422.008113:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767422.008116:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767422.008118:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099054720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767422.008122:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099054848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767422.008140:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767422.008150:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767422/real 1775767422] req@ffff8b23864f0700 x1862022099054848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767422.008162:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099054848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767422.008171:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767422.008174:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767422.008178:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099054848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767422.008185:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099054976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767422.008205:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767422.008214:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767422/real 1775767422] req@ffff8b23864f3b80 x1862022099054976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767422.008227:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099054976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767422.008233:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767422.008235:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767422.008238:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099054976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767422.903600:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767423.927669:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767424.951672:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767425.975641:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767426.999591:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767427.127597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767427.127608:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767427.127616:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767427.127627:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4888 00000100:00080000:2.0:1775767427.127641:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767427.127649:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767427.127703:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022099055104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767427.127731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767427.127734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767427.127736:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767427.127739:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4888 00000100:00080000:2.0:1775767427.127742:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767427.127745:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767427.127754:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767427.127767:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099055232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767427.127776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767427.127779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767427.127781:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767427.127784:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4888 00000100:00080000:2.0:1775767427.127787:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767427.127792:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767427.127796:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099055104:192.168.202.128@tcp:250:kworker.0 00000008:00100000:2.0:1775767427.127796:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767427.127805:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099055360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767427.127813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767427.127815:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767427.127816:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767427.127819:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4888 00000100:00080000:2.0:1775767427.127822:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767427.127824:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767427.127827:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767427.127834:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099055488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767427.127840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767427.127843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767427.127844:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767427.127846:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4888 00000100:00080000:2.0:1775767427.127849:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767427.127851:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767427.127854:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767427.127860:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099055616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767427.127887:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767427.127940:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767427/real 1775767427] req@ffff8b23b9ea4e00 x1862022099055104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767427.127962:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022099055104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767427.127974:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767427.127979:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767427.127983:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099055104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767427.127991:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099055232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767427.128017:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767427.128032:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767427/real 1775767427] req@ffff8b23b9ea4380 x1862022099055232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767427.128052:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099055232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767427.128061:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767427.128065:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767427.128068:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099055232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767427.128097:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099055360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767427.128121:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767427.128133:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767427/real 1775767427] req@ffff8b23b9ea7100 x1862022099055360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767427.128147:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099055360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767427.128156:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767427.128159:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767427.128163:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099055360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767427.128169:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099055488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767427.128190:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767427.128200:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767427/real 1775767427] req@ffff8b23b9ea6300 x1862022099055488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767427.128212:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099055488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767427.128227:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767427.128230:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767427.128234:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099055488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767427.128241:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099055616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767427.128261:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767427.128272:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767427/real 1775767427] req@ffff8b23b9ea7480 x1862022099055616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767427.128284:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099055616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767427.128317:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767427.128322:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767427.128325:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099055616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767428.023691:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767429.047583:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767430.071633:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767431.095693:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767432.119669:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767432.247565:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767432.247575:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767432.247580:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767432.247590:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4893 00000100:00080000:2.0:1775767432.247602:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767432.247609:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767432.247652:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099055744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767432.247679:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767432.247683:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767432.247684:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767432.247687:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4893 00000100:00080000:2.0:1775767432.247689:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767432.247692:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767432.247700:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767432.247713:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099055872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767432.247725:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767432.247729:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767432.247731:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767432.247738:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4893 00000100:00080000:2.0:1775767432.247744:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767432.247747:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099055744:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767432.247753:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767432.247766:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767432.247784:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099056000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767432.247794:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767432.247797:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767432.247799:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767432.247802:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4893 00000100:00080000:2.0:1775767432.247806:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767432.247809:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767432.247812:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767432.247833:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775767432.247837:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099056128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767432.247845:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767432.247848:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767432.247850:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767432.247855:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4893 00000100:00080000:2.0:1775767432.247857:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767432.247860:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767432.247865:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767432.247875:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099056256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767432.247884:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767432/real 1775767432] req@ffff8b23b9ea4700 x1862022099055744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767432.247900:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099055744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767432.247909:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767432.247912:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767432.247915:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099055744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767432.247927:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099055872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767432.247944:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767432.247952:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767432/real 1775767432] req@ffff8b23b9ea5880 x1862022099055872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767432.247959:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099055872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767432.247963:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767432.247965:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767432.247967:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099055872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767432.247985:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099056000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767432.247994:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767432.247999:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767432/real 1775767432] req@ffff8b23b9ea4000 x1862022099056000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767432.248005:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099056000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767432.248010:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767432.248011:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767432.248013:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099056000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767432.248017:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099056128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767432.248027:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767432.248031:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767432/real 1775767432] req@ffff8b23b9ea6a00 x1862022099056128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767432.248038:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099056128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767432.248042:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767432.248044:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767432.248045:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099056128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767432.248048:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099056256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767432.248057:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767432.248061:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767432/real 1775767432] req@ffff8b23b9ea4a80 x1862022099056256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767432.248067:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099056256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767432.248071:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767432.248073:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767432.248076:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099056256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767433.143631:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767434.167613:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767435.191744:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767436.215601:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767437.239645:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767437.367537:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767437.367547:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767437.367553:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767437.367563:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4898 00000100:00080000:2.0:1775767437.367577:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767437.367583:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767437.367627:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099056384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767437.367656:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767437.367659:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767437.367661:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767437.367664:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4898 00000100:00080000:2.0:1775767437.367666:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767437.367668:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767437.367683:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767437.367693:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099056512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767437.367703:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767437.367707:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767437.367709:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767437.367715:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4898 00000100:00080000:2.0:1775767437.367719:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767437.367727:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767437.367735:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099056384:192.168.202.128@tcp:250:kworker.0 00000008:00100000:2.0:1775767437.367738:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767437.367772:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099056640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767437.367793:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767437.367797:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767437.367800:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767437.367805:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4898 00000100:00080000:2.0:1775767437.367810:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775767437.367811:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775767437.367818:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767437.367824:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767437.367840:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099056768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767437.367848:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767437.367852:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767437.367855:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767437.367860:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4898 00000100:00080000:2.0:1775767437.367865:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767437.367869:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767437.367876:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767437/real 1775767437] req@ffff8b23b9ea6d80 x1862022099056384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1775767437.367881:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767437.367908:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099056896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767437.367915:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099056384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767437.367954:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767437.367960:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767437.367965:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099056384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767437.367973:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099056512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767437.367991:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767437.367999:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767437/real 1775767437] req@ffff8b23b9ea5c00 x1862022099056512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767437.368008:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099056512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767437.368014:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767437.368016:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767437.368018:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099056512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767437.368039:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099056640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767437.368051:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767437.368057:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767437/real 1775767437] req@ffff8b23b9ea7b80 x1862022099056640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767437.368064:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099056640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767437.368074:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767437.368077:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767437.368079:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099056640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767437.368082:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099056768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767437.368093:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767437.368098:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767437/real 1775767437] req@ffff8b23b9ea6680 x1862022099056768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767437.368105:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099056768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767437.368110:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767437.368112:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767437.368114:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099056768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767437.368117:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099056896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767437.368127:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767437.368133:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767437/real 1775767437] req@ffff8b23b9ea5f80 x1862022099056896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767437.368140:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099056896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767437.368144:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767437.368146:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767437.368149:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099056896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767438.263682:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767439.287688:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767440.311732:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767441.335728:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767442.359706:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767442.487586:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767442.487597:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767442.487602:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767442.487612:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4903 00000100:00080000:1.0:1775767442.487625:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767442.487631:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767442.487677:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099057024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767442.487703:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767442.487706:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767442.487707:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767442.487711:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4903 00000100:00080000:1.0:1775767442.487714:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767442.487719:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767442.487732:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767442.487749:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099057152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767442.487764:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767442.487765:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099057024:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767442.487771:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767442.487774:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767442.487779:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4903 00000100:00080000:1.0:1775767442.487782:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767442.487786:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767442.487791:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767442.487804:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099057280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767442.487811:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767442.487814:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767442.487816:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767442.487819:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4903 00000100:00080000:1.0:1775767442.487838:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767442.487840:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767442.487843:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767442.487849:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775767442.487853:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099057408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767442.487860:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767442.487863:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767442.487865:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767442.487868:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4903 00000100:00080000:1.0:1775767442.487871:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767442.487874:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767442.487877:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767442.487889:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099057536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767442.487899:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767442/real 1775767442] req@ffff8b23864fe680 x1862022099057024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767442.487919:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099057024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767442.487941:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767442.487949:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767442.487953:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099057024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767442.487983:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099057152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767442.488003:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767442.488013:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767442/real 1775767442] req@ffff8b23864fca80 x1862022099057152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767442.488021:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099057152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767442.488027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767442.488029:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767442.488032:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099057152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767442.488036:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099057280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767442.488056:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767442.488063:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767442/real 1775767442] req@ffff8b23864fea00 x1862022099057280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767442.488071:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099057280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767442.488076:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767442.488079:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767442.488081:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099057280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767442.488084:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099057408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767442.488096:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767442.488101:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767442/real 1775767442] req@ffff8b23864fd180 x1862022099057408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767442.488108:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099057408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767442.488113:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767442.488115:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767442.488117:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099057408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767442.488120:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099057536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767442.488130:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767442.488135:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767442/real 1775767442] req@ffff8b23864fce00 x1862022099057536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767442.488147:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099057536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767442.488166:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767442.488168:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767442.488170:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099057536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767443.383658:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767444.407662:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767445.431678:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767446.455695:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767447.479615:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767447.607562:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767447.607572:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767447.607577:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767447.607590:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4909 00000100:00080000:1.0:1775767447.607606:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767447.607614:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767447.607667:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099057664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767447.607701:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767447.607705:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767447.607707:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767447.607712:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4909 00000100:00080000:1.0:1775767447.607716:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767447.607721:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767447.607734:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767447.607755:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099057792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767447.607765:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099057664:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767447.607771:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767447.607775:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767447.607777:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767447.607781:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4909 00000100:00080000:1.0:1775767447.607784:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767447.607788:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767447.607793:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767447.607805:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099057920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767447.607812:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767447.607815:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767447.607817:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767447.607819:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4909 00000100:00080000:1.0:1775767447.607836:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767447.607839:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767447.607842:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767447.607850:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099058048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767447.607857:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767447.607859:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767447.607861:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767447.607864:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4909 00000100:00080000:1.0:1775767447.607867:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767447.607870:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767447.607874:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767447.607883:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099058176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767447.607902:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099057792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767447.607930:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099057920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767447.607946:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099058048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767447.607960:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099058176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767448.503628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767449.527615:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767450.551582:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775767450.679713:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775767450.679751:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775767450.679767:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767450.679818:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767450.679828:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767450.679837:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767450.679854:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775767450.679967:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767447/real 1775767450] req@ffff8b23864fc380 x1862022099057792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767450.680001:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099057792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767450.680014:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767450.680020:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767450.680025:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099057792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767450.680035:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767447/real 1775767450] req@ffff8b23864fdf80 x1862022099057920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767450.680057:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099057920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767450.680064:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767450.680069:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767450.680072:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099057920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767450.680091:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767447/real 1775767450] req@ffff8b23864ff800 x1862022099058048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767450.680103:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099058048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767450.680110:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767450.680115:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767450.680118:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099058048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767450.680125:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767447/real 1775767450] req@ffff8b23864fed80 x1862022099058176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767450.680138:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099058176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767450.680147:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767450.680150:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767450.680155:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099058176:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767450.680162:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767447/real 1775767450] req@ffff8b23864fc000 x1862022099057664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767450.680175:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099057664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767450.680182:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767450.680186:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767450.680189:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099057664:192.168.202.128@tcp:250:kworker.0 00010000:00010000:0.0:1775767451.575655:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767452.599619:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767452.727591:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767452.727602:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767452.727607:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767452.727627:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4914 00000100:00080000:2.0:1775767452.727638:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767452.727644:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767452.727678:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022099058304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767452.727720:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767452.727724:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767452.727726:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767452.727729:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4914 00000100:00080000:2.0:1775767452.727734:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767452.727738:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767452.727747:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767452.727763:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099058432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767452.727784:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767452.727788:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767452.727790:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767452.727795:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4914 00000100:00080000:2.0:1775767452.727799:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767452.727803:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767452.727809:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767452.727821:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099058560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767452.727830:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767452.727834:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767452.727836:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767452.727840:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4914 00000100:00080000:2.0:1775767452.727844:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767452.727848:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767452.727852:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767452.727861:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099058688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767452.727871:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767452.727874:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767452.727876:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767452.727880:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4914 00000100:00080000:2.0:1775767452.727883:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767452.727886:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767452.727891:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767452.727902:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099058816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767452.728013:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099058304:192.168.202.128@tcp:250:kworker.0 00000800:00000100:0.0:1775767452.728108:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767452.728186:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767452/real 1775767452] req@ffff8b23b9ea5500 x1862022099058304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767452.728208:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022099058304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767452.728220:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767452.728225:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767452.728229:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099058304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767452.728239:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099058432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767452.728265:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767452.728280:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767452/real 1775767452] req@ffff8b23b9ea5f80 x1862022099058432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767452.728300:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099058432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767452.728309:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767452.728313:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767452.728316:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099058432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767452.728322:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099058560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767452.728344:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767452.728355:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767452/real 1775767452] req@ffff8b23b9ea6680 x1862022099058560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767452.728366:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099058560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767452.728374:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767452.728377:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767452.728380:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099058560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767452.728385:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099058688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767452.728410:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767452.728422:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767452/real 1775767452] req@ffff8b23b9ea7b80 x1862022099058688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767452.728434:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099058688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767452.728448:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767452.728483:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767452.728488:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099058688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767452.728494:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099058816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767452.728516:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767452.728526:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767452/real 1775767452] req@ffff8b23b9ea5c00 x1862022099058816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767452.728536:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099058816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767452.728543:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767452.728546:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767452.728549:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099058816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767453.623595:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767454.647631:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767455.671606:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767456.695633:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767457.719657:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767457.847614:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767457.847626:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767457.847631:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767457.847641:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4919 00000100:00080000:0.0:1775767457.847654:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767457.847661:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767457.847706:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022099058944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767457.847742:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767457.847746:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767457.847747:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767457.847750:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4919 00000100:00080000:0.0:1775767457.847752:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767457.847755:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767457.847765:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767457.847775:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022099059072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767457.847784:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767457.847787:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767457.847788:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767457.847792:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4919 00000100:00080000:0.0:1775767457.847793:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767457.847797:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767457.847801:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767457.847806:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099058944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767457.847813:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099059200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767457.847822:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767457.847824:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767457.847825:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767457.847828:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4919 00000100:00080000:0.0:1775767457.847831:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767457.847833:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767457.847836:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767457.847844:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022099059328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767457.847851:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767457.847853:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767457.847854:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767457.847856:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4919 00000100:00080000:0.0:1775767457.847858:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767457.847861:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767457.847863:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767457.847870:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099059456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767457.847895:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767457.847965:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767457/real 1775767457] req@ffff8b2382dac000 x1862022099058944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767457.847986:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022099058944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767457.847996:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767457.848000:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767457.848004:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099058944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767457.848026:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099059072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767457.848055:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767457.848070:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767457/real 1775767457] req@ffff8b2382daf100 x1862022099059072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767457.848084:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022099059072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767457.848091:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767457.848093:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767457.848095:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099059072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767457.848121:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099059200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767457.848137:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767457.848144:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767457/real 1775767457] req@ffff8b2382daca80 x1862022099059200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767457.848152:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099059200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767457.848158:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767457.848160:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767457.848161:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099059200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767457.848165:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099059328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767457.848177:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767457.848182:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767457/real 1775767457] req@ffff8b2382dad880 x1862022099059328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767457.848195:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022099059328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767457.848199:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767457.848201:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767457.848203:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099059328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767457.848206:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099059456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767457.848216:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767457.848221:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767457/real 1775767457] req@ffff8b2382dac380 x1862022099059456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767457.848238:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099059456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767457.848244:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767457.848246:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767457.848248:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099059456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767458.743629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767459.767635:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767460.791672:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767461.815698:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767462.839744:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767462.967592:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767462.967602:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767462.967607:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767462.967618:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4924 00000100:00080000:2.0:1775767462.967630:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767462.967636:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767462.967686:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099059584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767462.967719:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767462.967724:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767462.967726:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767462.967731:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4924 00000100:00080000:2.0:1775767462.967744:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767462.967749:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767462.967760:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767462.967765:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099059584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775767462.967775:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099059712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767462.967786:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767462.967789:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767462.967791:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767462.967794:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4924 00000100:00080000:2.0:1775767462.967797:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767462.967800:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767462.967805:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767462.967814:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099059840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767462.967820:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767462.967823:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767462.967825:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767462.967828:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4924 00000100:00080000:2.0:1775767462.967836:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767462.967840:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775767462.967846:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:2.0:1775767462.967847:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767462.967856:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099059968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767462.967863:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767462.967866:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767462.967868:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767462.967871:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4924 00000100:00080000:2.0:1775767462.967874:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767462.967877:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767462.967880:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767462.967887:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099060096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767462.967918:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767462/real 1775767462] req@ffff8b23b9ea6d80 x1862022099059584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767462.967939:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099059584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767462.967970:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767462.967977:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767462.967983:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099059584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767462.968014:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099059712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767462.968036:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767462.968049:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767462/real 1775767462] req@ffff8b23b9ea4a80 x1862022099059712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767462.968063:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099059712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767462.968071:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767462.968075:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767462.968079:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099059712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767462.968086:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099059840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767462.968108:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767462.968120:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767462/real 1775767462] req@ffff8b23b9ea6a00 x1862022099059840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767462.968129:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099059840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767462.968135:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767462.968138:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767462.968140:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099059840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767462.968145:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099059968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767462.968161:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767462.968168:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767462/real 1775767462] req@ffff8b23b9ea4000 x1862022099059968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767462.968176:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099059968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767462.968181:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767462.968183:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767462.968184:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099059968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767462.968188:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099060096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767462.968199:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767462.968204:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767462/real 1775767462] req@ffff8b23b9ea5880 x1862022099060096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767462.968211:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099060096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767462.968216:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767462.968218:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767462.968219:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099060096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767463.863667:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767464.887613:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767465.911679:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767466.935666:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767467.959597:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767468.087523:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767468.087532:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767468.087536:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767468.087545:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4929 00000100:00080000:2.0:1775767468.087554:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767468.087560:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767468.087594:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099060224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767468.087620:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767468.087623:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767468.087625:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767468.087629:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4929 00000100:00080000:2.0:1775767468.087632:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767468.087636:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767468.087646:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767468.087658:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099060352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767468.087668:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767468.087672:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1775767468.087672:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099060224:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767468.087673:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767468.087677:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4929 00000100:00080000:2.0:1775767468.087680:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767468.087684:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767468.087689:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767468.087697:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099060480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767468.087711:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767468.087713:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767468.087715:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767468.087719:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4929 00000100:00080000:2.0:1775767468.087721:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767468.087724:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767468.087728:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767468.087735:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775767468.087736:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099060608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767468.087743:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767468.087746:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767468.087748:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767468.087751:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4929 00000100:00080000:2.0:1775767468.087754:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767468.087757:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767468.087761:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767468.087769:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099060736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767468.087791:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767468/real 1775767468] req@ffff8b23b9ea4700 x1862022099060224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767468.087811:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099060224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767468.087823:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767468.087826:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767468.087829:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099060224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767468.087848:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099060352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767468.087867:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767468.087876:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767468/real 1775767468] req@ffff8b23b9ea7480 x1862022099060352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767468.087884:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099060352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767468.087889:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767468.087891:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767468.087893:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099060352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767468.087897:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099060480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767468.087925:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767468.087933:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767468/real 1775767468] req@ffff8b23b9ea6300 x1862022099060480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767468.087941:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099060480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767468.087951:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767468.087953:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767468.087955:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099060480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767468.087958:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099060608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767468.087971:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767468.087977:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767468/real 1775767468] req@ffff8b23b9ea7100 x1862022099060608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767468.087984:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099060608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767468.087988:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767468.087990:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767468.087992:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099060608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767468.087995:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099060736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767468.088004:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767468.088009:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767468/real 1775767468] req@ffff8b23b9ea4380 x1862022099060736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767468.088015:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099060736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767468.088020:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767468.088022:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767468.088024:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099060736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767468.983590:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767470.007670:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767471.031573:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767472.055631:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767473.079569:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767473.207587:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767473.207597:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767473.207602:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767473.207613:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4934 00000100:00080000:0.0:1775767473.207626:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767473.207633:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767473.207673:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099060864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767473.207703:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767473.207707:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767473.207709:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767473.207713:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4934 00000100:00080000:0.0:1775767473.207718:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767473.207722:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767473.207732:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767473.207748:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099060992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767473.207752:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099060864:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767473.207770:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767473.207774:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767473.207776:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767473.207782:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4934 00000100:00080000:0.0:1775767473.207787:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767473.207792:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767473.207797:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767473.207813:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099061120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767473.207824:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775767473.207825:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767473.207831:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767473.207833:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767473.207839:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4934 00000100:00080000:0.0:1775767473.207853:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767473.207874:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767473.207881:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767473.207889:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767473/real 1775767473] req@ffff8b2382dae680 x1862022099060864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767473.207897:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099061248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767473.207913:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775767473.207917:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099060864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767473.207918:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767473.207922:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767473.207933:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4934 00000100:00080000:0.0:1775767473.207939:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767473.207939:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767473.207947:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:3.0:1775767473.207950:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000008:00100000:0.0:1775767473.207954:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767473.207956:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099060864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767473.207971:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099061376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767473.207989:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099060992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767473.208019:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767473.208032:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767473/real 1775767473] req@ffff8b2382dad500 x1862022099060992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767473.208042:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099060992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767473.208047:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767473.208050:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767473.208052:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099060992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767473.208056:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099061120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767473.208070:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767473.208077:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767473/real 1775767473] req@ffff8b2382daea00 x1862022099061120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767473.208085:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099061120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767473.208090:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767473.208092:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767473.208094:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099061120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767473.208098:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099061248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767473.208109:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767473.208114:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767473/real 1775767473] req@ffff8b2382daed80 x1862022099061248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767473.208122:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099061248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767473.208127:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767473.208129:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767473.208131:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099061248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767473.208134:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099061376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767473.208143:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767473.208147:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767473/real 1775767473] req@ffff8b2382daf480 x1862022099061376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767473.208155:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099061376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767473.208160:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767473.208162:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767473.208164:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099061376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767474.103628:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767475.127604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767476.151692:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767477.175611:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767478.199629:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767478.327577:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767478.327585:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767478.327590:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767478.327599:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4939 00000100:00080000:3.0:1775767478.327609:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767478.327615:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767478.327648:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099061504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767478.327672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767478.327676:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767478.327678:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767478.327682:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4939 00000100:00080000:3.0:1775767478.327686:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767478.327690:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767478.327701:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767478.327713:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099061504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767478.327717:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099061632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767478.327732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767478.327737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767478.327739:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767478.327743:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4939 00000100:00080000:3.0:1775767478.327747:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767478.327751:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767478.327755:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767478.327765:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099061760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767478.327771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767478.327775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1775767478.327776:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767478.327779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767478.327783:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4939 00000100:00080000:3.0:1775767478.327786:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767478.327789:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767478.327794:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767478.327801:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099061888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767478.327822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767478.327825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767478.327827:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767478.327830:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4939 00000100:00080000:3.0:1775767478.327833:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767478.327838:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767478/real 1775767478] req@ffff8b23864f2d80 x1862022099061504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767478.327839:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767478.327846:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767478.327858:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099062016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767478.327901:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099061504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767478.327910:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767478.327913:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767478.327916:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099061504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767478.327936:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099061632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767478.327954:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767478.327963:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767478/real 1775767478] req@ffff8b23864f3100 x1862022099061632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767478.327971:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099061632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767478.327977:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767478.327979:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767478.327982:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099061632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767478.327985:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099061760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767478.327995:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767478.328000:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767478/real 1775767478] req@ffff8b23864f1180 x1862022099061760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767478.328007:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099061760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767478.328016:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767478.328018:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767478.328020:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099061760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767478.328024:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099061888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767478.328035:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767478.328040:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767478/real 1775767478] req@ffff8b23864f2680 x1862022099061888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767478.328047:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099061888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767478.328052:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767478.328053:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767478.328055:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099061888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767478.328058:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099062016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767478.328068:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767478.328073:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767478/real 1775767478] req@ffff8b23864f0380 x1862022099062016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767478.328079:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099062016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767478.328084:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767478.328086:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767478.328088:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099062016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767479.223610:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767480.247618:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767481.271653:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767482.295633:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767483.319616:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767483.447568:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767483.447576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767483.447581:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767483.447591:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4944 00000100:00080000:2.0:1775767483.447603:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767483.447610:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767483.447654:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022099062144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767483.447682:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767483.447687:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767483.447688:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767483.447692:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4944 00000100:00080000:2.0:1775767483.447695:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767483.447698:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767483.447706:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767483.447724:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099062144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775767483.447730:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099062272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767483.447744:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767483.447749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767483.447751:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767483.447768:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4944 00000100:00080000:2.0:1775767483.447774:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767483.447779:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767483.447784:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767483.447794:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099062400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767483.447802:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775767483.447804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767483.447808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767483.447809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767483.447814:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4944 00000100:00080000:2.0:1775767483.447818:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767483.447821:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767483.447825:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767483.447832:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099062528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767483.447839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767483.447842:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767483/real 1775767483] req@ffff8b23b9ea4e00 x1862022099062144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767483.447843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767483.447846:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767483.447853:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4944 00000100:00080000:2.0:1775767483.447858:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767483.447861:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767483.447864:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767483.447868:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022099062144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767483.447876:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099062656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767483.447878:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767483.447886:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767483.447891:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099062144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767483.447917:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099062272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767483.447949:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767483.447963:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767483/real 1775767483] req@ffff8b23b9ea4380 x1862022099062272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767483.447976:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099062272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767483.447988:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767483.447991:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767483.447993:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099062272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767483.447998:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099062400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767483.448014:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767483.448021:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767483/real 1775767483] req@ffff8b23b9ea7100 x1862022099062400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767483.448029:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099062400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767483.448034:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767483.448036:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767483.448038:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099062400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767483.448041:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099062528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767483.448052:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767483.448058:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767483/real 1775767483] req@ffff8b23b9ea6300 x1862022099062528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767483.448065:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099062528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767483.448070:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767483.448072:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767483.448073:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099062528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767483.448077:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099062656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767483.448087:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767483.448093:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767483/real 1775767483] req@ffff8b23b9ea7480 x1862022099062656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767483.448104:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099062656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767483.448109:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767483.448111:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767483.448113:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099062656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767484.343625:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767485.367588:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767486.391609:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767487.415675:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767488.439628:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767488.567556:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767488.567566:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767488.567571:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767488.567582:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4949 00000100:00080000:2.0:1775767488.567594:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767488.567602:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767488.567655:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099062784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767488.567683:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767488.567691:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767488.567693:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767488.567697:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4949 00000100:00080000:2.0:1775767488.567701:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767488.567706:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767488.567730:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767488.567749:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099062784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775767488.567757:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099062912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767488.567776:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767488.567779:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767488.567781:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767488.567785:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4949 00000100:00080000:2.0:1775767488.567789:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767488.567793:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767488.567797:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767488.567808:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099063040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767488.567821:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767488.567823:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767488.567825:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767488.567829:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4949 00000100:00080000:2.0:1775767488.567835:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775767488.567836:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775767488.567840:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767488.567844:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767488.567852:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099063168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767488.567860:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767488.567863:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767488.567864:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767488.567866:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4949 00000100:00080000:2.0:1775767488.567868:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767488.567871:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767488.567879:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767488.567886:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767488/real 1775767488] req@ffff8b23b9ea4700 x1862022099062784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767488.567892:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099063296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767488.567928:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099062784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767488.567937:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767488.567941:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767488.567944:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099062784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767488.567966:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099062912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767488.567982:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767488.567991:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767488/real 1775767488] req@ffff8b23b9ea5880 x1862022099062912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767488.568004:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099062912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767488.568010:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767488.568012:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767488.568015:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099062912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767488.568018:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099063040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767488.568029:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767488.568034:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767488/real 1775767488] req@ffff8b23b9ea4000 x1862022099063040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767488.568042:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099063040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767488.568047:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767488.568049:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767488.568050:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099063040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767488.568054:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099063168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767488.568066:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767488.568071:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767488/real 1775767488] req@ffff8b23b9ea6a00 x1862022099063168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767488.568079:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099063168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767488.568088:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767488.568091:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767488.568093:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099063168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767488.568096:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099063296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767488.568105:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767488.568110:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767488/real 1775767488] req@ffff8b23b9ea4a80 x1862022099063296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767488.568117:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099063296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767488.568122:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767488.568124:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767488.568126:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099063296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767489.463651:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767490.487636:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767491.511727:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767492.535611:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767493.559711:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767493.687577:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767493.687590:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767493.687596:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767493.687608:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4955 00000100:00080000:0.0:1775767493.687622:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767493.687630:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767493.687721:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022099063424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767493.687768:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767493.687773:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767493.687775:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767493.687780:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4955 00000100:00080000:0.0:1775767493.687784:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767493.687788:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767493.687801:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767493.687815:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022099063552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767493.687826:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099063424:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767493.687830:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767493.687835:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767493.687837:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767493.687843:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4955 00000100:00080000:0.0:1775767493.687846:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767493.687850:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767493.687855:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767493.687869:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022099063680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767493.687879:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767493.687884:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767493.687885:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767493.687890:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4955 00000100:00080000:0.0:1775767493.687895:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767493.687901:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775767493.687901:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:0.0:1775767493.687909:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767493.687921:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022099063808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767493.687932:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767493.687935:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767493.687937:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767493.687941:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4955 00000100:00080000:0.0:1775767493.687946:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767493.687951:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767493/real 1775767493] req@ffff8b2382dac700 x1862022099063424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767493.687953:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767493.687962:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767493.687973:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022099063424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767493.687979:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099063936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767493.688000:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767493.688005:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767493.688010:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099063424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767493.688037:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099063552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767493.688054:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767493.688063:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767493/real 1775767493] req@ffff8b2382dace00 x1862022099063552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767493.688071:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022099063552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767493.688077:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767493.688079:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767493.688081:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099063552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767493.688085:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099063680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767493.688099:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767493.688105:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767493/real 1775767493] req@ffff8b2382dadf80 x1862022099063680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767493.688112:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022099063680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767493.688118:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767493.688120:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767493.688122:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099063680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767493.688126:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099063808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767493.688137:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767493.688142:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767493/real 1775767493] req@ffff8b2382dafb80 x1862022099063808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767493.688150:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022099063808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767493.688155:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767493.688157:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767493.688158:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099063808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767493.688162:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099063936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767493.688171:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767493.688176:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767493/real 1775767493] req@ffff8b2382daf480 x1862022099063936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767493.688188:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099063936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767493.688194:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767493.688196:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767493.688197:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099063936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767494.583607:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767495.607726:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767496.631671:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767497.655741:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767498.679744:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767498.807564:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767498.807572:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767498.807576:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767498.807585:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4960 00000100:00080000:0.0:1775767498.807595:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767498.807601:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767498.807667:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099064064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767498.807690:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767498.807693:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767498.807710:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767498.807713:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4960 00000100:00080000:0.0:1775767498.807716:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767498.807719:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767498.807728:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767498.807738:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099064064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767498.807740:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099064192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767498.807753:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767498.807756:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767498.807757:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767498.807761:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4960 00000100:00080000:0.0:1775767498.807763:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767498.807766:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767498.807768:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767498.807780:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099064320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767498.807786:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767498.807789:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767498.807790:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767498.807793:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4960 00000100:00080000:0.0:1775767498.807796:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767498.807799:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775767498.807803:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:0.0:1775767498.807804:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767498.807825:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099064448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767498.807839:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767498/real 1775767498] req@ffff8b2382daed80 x1862022099064064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767498.807843:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767498.807871:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767498.807874:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767498.807880:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4960 00000100:00100000:3.0:1775767498.807883:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099064064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767498.807888:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767498.807897:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:3.0:1775767498.807898:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1775767498.807904:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775767498.807907:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767498.807913:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099064064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767498.807919:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099064576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767498.807941:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099064192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767498.807970:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767498.807981:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767498/real 1775767498] req@ffff8b2382daea00 x1862022099064192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767498.807989:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099064192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767498.808000:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767498.808002:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767498.808005:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099064192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767498.808009:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099064320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767498.808020:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767498.808026:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767498/real 1775767498] req@ffff8b2382dad500 x1862022099064320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767498.808033:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099064320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767498.808038:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767498.808039:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767498.808041:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099064320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767498.808045:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099064448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767498.808055:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767498.808060:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767498/real 1775767498] req@ffff8b2382dae680 x1862022099064448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767498.808067:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099064448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767498.808071:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767498.808073:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767498.808076:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099064448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767498.808079:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099064576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767498.808089:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767498.808093:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767498/real 1775767498] req@ffff8b2382dac380 x1862022099064576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767498.808100:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099064576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767498.808105:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767498.808107:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767498.808108:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099064576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767499.703676:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767500.727718:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767501.751836:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767502.775695:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767503.799689:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767503.927656:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767503.927666:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767503.927671:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767503.927682:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4965 00000100:00080000:2.0:1775767503.927695:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767503.927701:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767503.927749:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099064704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767503.927777:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767503.927782:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767503.927784:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767503.927787:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4965 00000100:00080000:2.0:1775767503.927790:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767503.927792:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767503.927802:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767503.927812:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099064832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767503.927821:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767503.927824:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767503.927825:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767503.927829:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4965 00000100:00080000:2.0:1775767503.927832:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767503.927835:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767503.927838:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767503.927845:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099064960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767503.927852:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767503.927855:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767503.927857:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767503.927862:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4965 00000100:00080000:2.0:1775767503.927867:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767503.927871:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099064704:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767503.927872:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767503.927889:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767503.927897:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099065088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767503.927904:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767503.927907:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767503.927910:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767503.927914:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4965 00000100:00080000:2.0:1775767503.927918:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767503.927923:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767503.927929:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767503.927943:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099065216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767503.927966:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767503.928017:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767503/real 1775767503] req@ffff8b23b9ea6d80 x1862022099064704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767503.928038:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099064704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767503.928049:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767503.928054:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767503.928059:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099064704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767503.928067:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099064832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767503.928092:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767503.928106:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767503/real 1775767503] req@ffff8b23b9ea5c00 x1862022099064832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767503.928119:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099064832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767503.928127:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767503.928130:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767503.928133:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099064832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767503.928161:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099064960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767503.928196:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767503.928215:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767503/real 1775767503] req@ffff8b23b9ea7b80 x1862022099064960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767503.928224:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099064960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767503.928229:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767503.928232:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767503.928234:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099064960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767503.928238:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099065088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767503.928254:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767503.928259:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767503/real 1775767503] req@ffff8b23b9ea6680 x1862022099065088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767503.928266:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099065088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767503.928272:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767503.928274:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767503.928276:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099065088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767503.928279:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099065216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767503.928289:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767503.928294:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767503/real 1775767503] req@ffff8b23b9ea5f80 x1862022099065216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767503.928300:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099065216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767503.928305:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767503.928307:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767503.928309:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099065216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767504.823682:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767505.847707:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767506.871665:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767507.895623:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767508.919719:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767509.047625:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767509.047635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767509.047640:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767509.047652:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4970 00000100:00080000:3.0:1775767509.047665:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767509.047672:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767509.047715:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099065344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767509.047746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767509.047751:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767509.047756:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767509.047759:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4970 00000100:00080000:3.0:1775767509.047762:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767509.047765:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767509.047775:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767509.047784:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099065472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767509.047795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767509.047798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767509.047801:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767509.047803:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099065344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767509.047806:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4970 00000100:00080000:3.0:1775767509.047811:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767509.047815:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767509.047818:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767509.047825:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099065600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767509.047832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767509.047835:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767509.047837:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767509.047841:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4970 00000100:00080000:3.0:1775767509.047845:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767509.047849:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767509.047866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767509.047882:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767509.047887:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099065728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767509.047899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767509.047903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767509.047905:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767509.047909:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4970 00000100:00080000:3.0:1775767509.047913:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767509.047916:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767509.047921:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767509.047931:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767509/real 1775767509] req@ffff8b23864f1c00 x1862022099065344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767509.047934:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099065856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767509.047973:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099065344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767509.047984:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767509.047988:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767509.047991:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099065344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767509.048015:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099065472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767509.048032:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767509.048041:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767509/real 1775767509] req@ffff8b23864f1f80 x1862022099065472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767509.048049:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099065472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767509.048055:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767509.048057:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767509.048059:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099065472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767509.048063:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099065600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767509.048073:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767509.048078:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767509/real 1775767509] req@ffff8b23864f0a80 x1862022099065600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767509.048085:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099065600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767509.048090:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767509.048092:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767509.048094:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099065600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767509.048099:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099065728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767509.048114:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767509.048119:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767509/real 1775767509] req@ffff8b23864f0380 x1862022099065728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767509.048127:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099065728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767509.048132:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767509.048134:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767509.048135:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099065728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767509.048139:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099065856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767509.048149:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767509.048154:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767509/real 1775767509] req@ffff8b23864f2680 x1862022099065856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767509.048161:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099065856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767509.048166:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767509.048168:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767509.048170:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099065856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767509.943644:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767510.967784:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767511.991640:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767513.015701:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767514.039725:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767514.167607:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767514.167615:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767514.167619:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767514.167628:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4975 00000100:00080000:3.0:1775767514.167638:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767514.167644:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767514.167678:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099065984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767514.167709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767514.167712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767514.167713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767514.167715:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4975 00000100:00080000:3.0:1775767514.167719:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767514.167722:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767514.167729:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767514.167739:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099066112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767514.167748:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767514.167751:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767514.167753:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767514.167756:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099065984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767514.167758:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4975 00000100:00080000:3.0:1775767514.167763:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767514.167769:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767514.167773:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767514.167781:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099066240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767514.167787:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767514.167790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767514.167792:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767514.167796:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4975 00000100:00080000:3.0:1775767514.167800:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767514.167803:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767514.167808:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767514.167818:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099066368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767514.167835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767514.167838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767514.167840:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767514.167844:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4975 00000100:00080000:3.0:1775767514.167849:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767514.167853:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767514.167859:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767514.167877:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099066112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767514.167884:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099066496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767514.167916:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099066240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767514.167935:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099066368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767514.167962:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099066496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767515.063746:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767516.087652:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767517.111687:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767517.239927:0:2363:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775767517.240002:0:2363:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767517.240031:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767517.240100:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767517.240119:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767517.240151:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767517.240168:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:3.0:1775767517.240185:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767514/real 1775767517] req@ffff8b23864f3100 x1862022099066112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767517.240231:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099066112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767517.240252:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767517.240260:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767517.240267:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099066112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767517.240299:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767514/real 1775767517] req@ffff8b23864f2d80 x1862022099066240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767517.240319:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099066240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767517.240332:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767517.240338:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767517.240343:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099066240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767517.240354:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767514/real 1775767517] req@ffff8b23864f3b80 x1862022099066368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767517.240371:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099066368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767517.240382:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767517.240387:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767517.240392:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099066368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767517.240403:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767514/real 1775767517] req@ffff8b23864f1180 x1862022099065984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767517.240425:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099065984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767517.240458:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767517.240464:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767517.240501:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099065984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767517.240556:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767514/real 1775767517] req@ffff8b23864f0700 x1862022099066496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767517.240573:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099066496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767517.240585:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767517.240592:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767517.240596:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099066496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767518.135703:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767519.159654:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767519.287559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767519.287571:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767519.287575:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767519.287587:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4980 00000100:00080000:3.0:1775767519.287600:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767519.287607:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767519.287649:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099066624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767519.287678:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767519.287681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767519.287682:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767519.287685:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4980 00000100:00080000:3.0:1775767519.287688:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767519.287691:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767519.287700:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767519.287709:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099066752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767519.287718:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099066624:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767519.287721:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767519.287727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767519.287729:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767519.287734:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4980 00000100:00080000:3.0:1775767519.287739:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767519.287745:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767519.287751:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767519.287764:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099066880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767519.287775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767519.287780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767519.287782:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767519.287787:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4980 00000100:00080000:3.0:1775767519.287791:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767519.287795:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767519.287799:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767519.287810:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767519.287814:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099067008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767519.287826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767519.287830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767519.287832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767519.287837:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4980 00000100:00080000:3.0:1775767519.287842:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767519.287846:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767519.287852:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767519.287865:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099067136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767519.287874:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767519/real 1775767519] req@ffff8b23864f0700 x1862022099066624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767519.287914:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099066624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767519.287929:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767519.287935:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767519.287939:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099066624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767519.287980:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099066752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767519.288006:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767519.288022:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767519/real 1775767519] req@ffff8b23864f1180 x1862022099066752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767519.288039:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099066752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767519.288051:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767519.288056:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767519.288060:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099066752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767519.288066:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099066880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767519.288089:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767519.288101:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767519/real 1775767519] req@ffff8b23864f3b80 x1862022099066880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767519.288113:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099066880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767519.288121:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767519.288125:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767519.288128:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099066880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767519.288135:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099067008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767519.288160:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767519.288174:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767519/real 1775767519] req@ffff8b23864f2d80 x1862022099067008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767519.288185:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099067008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767519.288193:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767519.288196:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767519.288200:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099067008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767519.288205:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099067136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767519.288229:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767519.288240:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767519/real 1775767519] req@ffff8b23864f3100 x1862022099067136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767519.288253:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099067136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767519.288262:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767519.288265:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767519.288268:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099067136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767520.183717:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767521.207727:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767522.231668:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767523.255665:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767524.279644:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767524.407597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767524.407608:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767524.407614:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767524.407625:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4985 00000100:00080000:3.0:1775767524.407639:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767524.407646:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767524.407697:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099067264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767524.407734:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767524.407739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767524.407741:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767524.407746:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4985 00000100:00080000:3.0:1775767524.407749:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767524.407753:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767524.407763:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767524.407778:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099067392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767524.407788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767524.407791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767524.407792:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767524.407796:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4985 00000100:00080000:3.0:1775767524.407798:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767524.407803:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099067264:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767524.407804:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767524.407811:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767524.407843:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099067520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767524.407867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767524.407871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767524.407873:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767524.407878:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4985 00000100:00080000:3.0:1775767524.407883:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767524.407888:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767524.407893:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767524.407896:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767524.407908:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099067648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767524.407917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767524.407920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767524.407922:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767524.407925:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4985 00000100:00080000:3.0:1775767524.407928:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767524.407930:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767524.407934:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767524.407942:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099067776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767524.407953:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767524/real 1775767524] req@ffff8b23864f3480 x1862022099067264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767524.407972:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099067264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767524.407983:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767524.407986:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767524.407990:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099067264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767524.407996:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099067392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767524.408015:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767524.408024:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767524/real 1775767524] req@ffff8b23864f2a00 x1862022099067392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767524.408033:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099067392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767524.408038:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767524.408041:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767524.408043:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099067392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767524.408068:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099067520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767524.408078:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767524.408084:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767524/real 1775767524] req@ffff8b23864f0000 x1862022099067520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767524.408101:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099067520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767524.408107:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767524.408109:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767524.408111:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099067520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767524.408114:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099067648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767524.408126:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767524.408131:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767524/real 1775767524] req@ffff8b23864f3100 x1862022099067648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767524.408143:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099067648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767524.408148:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767524.408150:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767524.408152:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099067648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767524.408155:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099067776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767524.408165:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767524.408170:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767524/real 1775767524] req@ffff8b23864f2d80 x1862022099067776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767524.408176:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099067776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767524.408190:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767524.408192:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767524.408193:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099067776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767525.303610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767526.327624:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767527.351617:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767528.375679:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767529.399611:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767529.527547:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767529.527558:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767529.527563:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767529.527574:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4990 00000100:00080000:2.0:1775767529.527586:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767529.527593:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767529.527631:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022099067904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767529.527655:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767529.527658:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767529.527660:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767529.527662:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4990 00000100:00080000:2.0:1775767529.527666:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767529.527669:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767529.527677:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767529.527687:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099068032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767529.527696:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767529.527699:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767529.527701:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767529.527704:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4990 00000100:00080000:2.0:1775767529.527707:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767529.527710:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099067904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767529.527711:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767529.527717:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767529.527725:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099068160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767529.527731:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767529.527734:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767529.527735:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767529.527737:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4990 00000100:00080000:2.0:1775767529.527740:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767529.527743:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767529.527746:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767529.527753:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099068288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767529.527765:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767529.527768:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767529.527770:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767529.527773:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4990 00000800:00000100:0.0:1775767529.527779:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775767529.527781:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767529.527787:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767529.527793:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767529.527806:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099068416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767529.527823:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767529/real 1775767529] req@ffff8b23b9ea5500 x1862022099067904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767529.527851:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022099067904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767529.527862:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767529.527866:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767529.527871:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099067904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767529.527878:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099068032:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767529.527901:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767529.527914:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767529/real 1775767529] req@ffff8b23b9ea5f80 x1862022099068032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767529.527927:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099068032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767529.527935:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767529.527939:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767529.527942:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099068032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767529.527965:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099068160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767529.527984:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767529.528004:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767529/real 1775767529] req@ffff8b23b9ea6680 x1862022099068160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767529.528016:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099068160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767529.528030:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767529.528033:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767529.528037:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099068160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767529.528042:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099068288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767529.528061:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767529.528070:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767529/real 1775767529] req@ffff8b23b9ea7b80 x1862022099068288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767529.528080:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099068288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767529.528088:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767529.528091:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767529.528094:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099068288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767529.528100:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099068416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767529.528118:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767529.528128:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767529/real 1775767529] req@ffff8b23b9ea5c00 x1862022099068416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767529.528139:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099068416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767529.528146:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767529.528149:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767529.528153:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099068416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767530.423625:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767531.447607:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767532.471680:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767533.495675:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767534.519625:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767534.647530:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767534.647540:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767534.647545:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767534.647555:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 4996 00000100:00080000:2.0:1775767534.647566:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767534.647573:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767534.647609:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099068544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767534.647639:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767534.647643:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767534.647645:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767534.647649:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 4996 00000100:00080000:2.0:1775767534.647654:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767534.647657:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767534.647666:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767534.647678:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099068672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767534.647691:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767534.647694:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767534.647696:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767534.647702:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4996 00000100:00080000:2.0:1775767534.647705:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767534.647711:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767534.647719:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767534.647721:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099068544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775767534.647737:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099068800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767534.647751:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767534.647755:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767534.647757:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767534.647762:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4996 00000100:00080000:2.0:1775767534.647767:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767534.647771:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767534.647778:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767534.647793:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099068928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767534.647804:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775767534.647812:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775767534.647826:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767534.647830:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767534.647837:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 4996 00000100:00080000:2.0:1775767534.647842:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767534.647848:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767534.647854:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767534.647866:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767534/real 1775767534] req@ffff8b23b9ea6d80 x1862022099068544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767534.647876:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099069056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767534.647895:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099068544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767534.647910:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767534.647917:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767534.647922:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099068544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767534.647931:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099068672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767534.647957:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767534.647968:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767534/real 1775767534] req@ffff8b23b9ea4a80 x1862022099068672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767534.647977:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099068672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767534.647982:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767534.647984:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767534.647987:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099068672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767534.648012:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099068800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767534.648025:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767534.648032:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767534/real 1775767534] req@ffff8b23b9ea6a00 x1862022099068800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767534.648040:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099068800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767534.648045:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767534.648047:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767534.648048:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099068800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767534.648053:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099068928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767534.648065:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767534.648071:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767534/real 1775767534] req@ffff8b23b9ea4000 x1862022099068928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767534.648078:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099068928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767534.648083:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767534.648085:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767534.648087:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099068928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767534.648090:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099069056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767534.648100:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767534.648106:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767534/real 1775767534] req@ffff8b23b9ea5880 x1862022099069056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767534.648118:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099069056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767534.648123:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767534.648125:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767534.648126:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099069056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767535.543667:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767536.567661:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767537.591645:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767538.615672:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767539.639659:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767539.767603:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767539.767617:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767539.767624:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767539.767638:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5001 00000100:00080000:3.0:1775767539.767655:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767539.767663:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767539.767717:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099069184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767539.767750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767539.767754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767539.767756:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767539.767760:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5001 00000100:00080000:3.0:1775767539.767764:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767539.767769:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767539.767783:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767539.767800:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099069312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767539.767814:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099069184:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767539.767815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767539.767823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767539.767826:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767539.767831:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5001 00000100:00080000:3.0:1775767539.767834:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767539.767837:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767539.767842:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767539.767854:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099069440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767539.767866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767539.767869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767539.767871:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767539.767873:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5001 00000100:00080000:3.0:1775767539.767876:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767539.767878:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767539.767881:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767539.767888:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099069568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767539.767894:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767539.767897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1775767539.767901:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767539.767902:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767539.767909:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5001 00000100:00080000:3.0:1775767539.767914:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767539.767920:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767539.767928:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767539.767942:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099069696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767539.767951:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767539/real 1775767539] req@ffff8b23864f3b80 x1862022099069184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767539.767970:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099069184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767539.767979:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767539.767984:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767539.767987:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099069184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767539.768020:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099069312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767539.768036:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767539.768046:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767539/real 1775767539] req@ffff8b23864f1180 x1862022099069312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767539.768059:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099069312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767539.768065:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767539.768068:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767539.768070:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099069312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767539.768075:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099069440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767539.768087:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767539.768093:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767539/real 1775767539] req@ffff8b23864f0700 x1862022099069440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767539.768101:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099069440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767539.768107:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767539.768108:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767539.768110:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099069440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767539.768113:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099069568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767539.768125:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767539.768131:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767539/real 1775767539] req@ffff8b23864f2680 x1862022099069568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767539.768143:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099069568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767539.768158:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767539.768162:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767539.768166:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099069568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767539.768173:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099069696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767539.768195:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767539.768206:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767539/real 1775767539] req@ffff8b23864f0380 x1862022099069696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767539.768227:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099069696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767539.768235:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767539.768238:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767539.768241:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099069696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767540.663652:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767541.687618:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767542.711600:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767543.735584:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767544.759628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767544.887511:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767544.887519:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767544.887523:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767544.887531:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5006 00000100:00080000:3.0:1775767544.887541:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767544.887547:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767544.887578:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099069824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767544.887602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767544.887605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767544.887607:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767544.887611:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5006 00000100:00080000:3.0:1775767544.887615:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767544.887619:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767544.887628:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767544.887642:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099069952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767544.887654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767544.887658:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767544.887660:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767544.887665:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5006 00000100:00080000:3.0:1775767544.887670:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767544.887673:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767544.887677:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767544.887688:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099070080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767544.887690:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099069824:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767544.887704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767544.887707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767544.887708:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767544.887711:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5006 00000100:00080000:3.0:1775767544.887714:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767544.887716:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767544.887719:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767544.887731:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099070208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767544.887741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767544.887744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767544.887746:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767544.887750:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5006 00000800:00000100:0.0:1775767544.887755:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767544.887757:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767544.887763:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767544.887769:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767544.887783:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099070336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767544.887793:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767544/real 1775767544] req@ffff8b23864f0a80 x1862022099069824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767544.887814:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099069824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767544.887825:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767544.887830:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767544.887835:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099069824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767544.887843:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099069952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767544.887868:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767544.887880:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767544/real 1775767544] req@ffff8b23864f1f80 x1862022099069952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767544.887888:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099069952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767544.887893:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767544.887896:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767544.887898:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099069952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767544.887917:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099070080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767544.887933:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767544.887943:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767544/real 1775767544] req@ffff8b23864f1c00 x1862022099070080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767544.887955:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099070080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767544.887964:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767544.887967:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767544.887970:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099070080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767544.887976:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099070208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767544.887997:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767544.888008:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767544/real 1775767544] req@ffff8b23864f0380 x1862022099070208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767544.888020:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099070208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767544.888028:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767544.888031:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767544.888035:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099070208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767544.888041:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099070336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767544.888060:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767544.888069:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767544/real 1775767544] req@ffff8b23864f2680 x1862022099070336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767544.888076:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099070336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767544.888082:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767544.888084:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767544.888086:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099070336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767545.783609:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767546.807586:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767547.831592:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767548.855587:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767549.879666:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767550.007546:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767550.007554:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767550.007558:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767550.007566:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5011 00000100:00080000:2.0:1775767550.007576:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767550.007581:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767550.007616:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099070464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767550.007639:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767550.007642:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767550.007643:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767550.007645:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5011 00000100:00080000:2.0:1775767550.007647:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767550.007650:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767550.007656:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767550.007663:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099070592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767550.007676:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767550.007679:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767550.007681:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767550.007685:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5011 00000100:00080000:2.0:1775767550.007687:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767550.007689:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767550.007692:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767550.007698:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099070720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767550.007705:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767550.007710:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1775767550.007712:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099070464:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767550.007712:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767550.007715:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5011 00000100:00080000:2.0:1775767550.007719:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767550.007723:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767550.007726:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767550.007733:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099070848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767550.007739:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767550.007742:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767550.007743:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767550.007745:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5011 00000100:00080000:2.0:1775767550.007748:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767550.007750:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767550.007753:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767550.007765:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099070976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767550.007793:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767550.007835:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767550/real 1775767550] req@ffff8b23b9ea4700 x1862022099070464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767550.007853:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099070464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767550.007864:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767550.007868:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767550.007872:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099070464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767550.007879:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099070592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767550.007901:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767550.007911:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767550/real 1775767550] req@ffff8b23b9ea7480 x1862022099070592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767550.007919:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099070592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767550.007925:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767550.007927:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767550.007929:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099070592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767550.007948:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099070720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767550.007959:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767550.007965:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767550/real 1775767550] req@ffff8b23b9ea6300 x1862022099070720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767550.007972:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099070720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767550.007982:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767550.007984:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767550.007986:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099070720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767550.007990:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099070848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767550.008000:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767550.008005:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767550/real 1775767550] req@ffff8b23b9ea7100 x1862022099070848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767550.008012:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099070848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767550.008017:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767550.008018:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767550.008020:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099070848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767550.008023:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099070976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767550.008033:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767550.008038:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767550/real 1775767550] req@ffff8b23b9ea4380 x1862022099070976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767550.008044:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099070976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767550.008049:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767550.008051:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767550.008053:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099070976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767550.903645:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767551.927651:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767552.951657:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767553.975649:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767554.999690:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767555.127589:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767555.127603:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767555.127608:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767555.127619:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5016 00000100:00080000:2.0:1775767555.127633:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767555.127639:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767555.127685:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022099071104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767555.127716:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767555.127719:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767555.127721:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767555.127723:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5016 00000100:00080000:2.0:1775767555.127726:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767555.127728:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767555.127738:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767555.127748:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099071232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767555.127756:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767555.127762:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767555.127763:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767555.127767:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5016 00000100:00080000:2.0:1775767555.127770:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767555.127772:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767555.127776:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767555.127783:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099071360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767555.127789:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767555.127792:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767555.127793:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767555.127796:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5016 00000100:00080000:2.0:1775767555.127799:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767555.127801:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767555.127805:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767555.127811:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099071488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767555.127818:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767555.127824:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767555.127825:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767555.127827:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099071104:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767555.127831:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5016 00000100:00080000:2.0:1775767555.127839:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767555.127845:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767555.127850:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767555.127862:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099071616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767555.127907:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767555.127946:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767555/real 1775767555] req@ffff8b23b9ea4e00 x1862022099071104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767555.127960:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022099071104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767555.127968:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767555.127971:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767555.127974:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099071104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767555.127981:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099071232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767555.127996:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767555.128004:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767555/real 1775767555] req@ffff8b23b9ea4380 x1862022099071232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767555.128012:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099071232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767555.128018:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767555.128020:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767555.128023:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099071232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767555.128026:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099071360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767555.128038:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767555.128044:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767555/real 1775767555] req@ffff8b23b9ea7100 x1862022099071360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767555.128051:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099071360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767555.128056:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767555.128058:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767555.128060:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099071360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767555.128096:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099071488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767555.128111:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767555.128118:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767555/real 1775767555] req@ffff8b23b9ea6300 x1862022099071488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767555.128126:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099071488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767555.128131:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767555.128133:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767555.128135:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099071488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767555.128139:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099071616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767555.128148:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767555.128154:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767555/real 1775767555] req@ffff8b23b9ea7480 x1862022099071616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767555.128166:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099071616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767555.128171:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767555.128173:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767555.128175:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099071616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767556.023718:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767557.047655:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767558.071686:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767559.095586:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767560.119685:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767560.247631:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767560.247644:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767560.247649:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767560.247661:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5021 00000100:00080000:0.0:1775767560.247675:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767560.247681:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767560.247730:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022099071744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767560.247761:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767560.247764:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767560.247765:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767560.247768:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5021 00000100:00080000:0.0:1775767560.247770:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767560.247773:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767560.247782:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767560.247794:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099071872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767560.247803:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767560.247806:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767560.247807:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767560.247810:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5021 00000100:00080000:0.0:1775767560.247812:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767560.247816:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767560.247820:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767560.247830:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022099072000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767560.247832:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099071744:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767560.247839:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767560.247842:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767560.247844:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767560.247846:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5021 00000100:00080000:0.0:1775767560.247848:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767560.247850:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767560.247853:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767560.247861:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022099072128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767560.247868:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767560.247871:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767560.247872:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767560.247875:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5021 00000100:00080000:0.0:1775767560.247878:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767560.247881:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767560.247884:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767560.247891:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099072256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767560.247923:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767560.247985:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767560/real 1775767560] req@ffff8b2382dad880 x1862022099071744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767560.248009:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022099071744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767560.248059:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767560.248064:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767560.248069:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099071744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767560.248077:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099071872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767560.248104:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767560.248118:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767560/real 1775767560] req@ffff8b2382daca80 x1862022099071872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767560.248133:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099071872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767560.248143:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767560.248147:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767560.248151:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099071872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767560.248183:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099072000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767560.248206:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767560.248218:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767560/real 1775767560] req@ffff8b2382daf100 x1862022099072000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767560.248231:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022099072000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767560.248240:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767560.248244:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767560.248247:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099072000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767560.248254:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099072128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767560.248275:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767560.248286:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767560/real 1775767560] req@ffff8b2382dac000 x1862022099072128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767560.248298:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022099072128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767560.248307:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767560.248310:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767560.248314:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099072128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767560.248320:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099072256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767560.248340:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767560.248351:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767560/real 1775767560] req@ffff8b2382dac380 x1862022099072256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767560.248363:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099072256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767560.248372:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767560.248375:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767560.248378:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099072256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767561.143682:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767562.167732:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767563.191628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767564.215664:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767565.239731:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767565.367608:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767565.367623:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767565.367630:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767565.367644:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5026 00000100:00080000:3.0:1775767565.367660:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767565.367669:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767565.367722:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099072384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767565.367758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767565.367763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767565.367765:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767565.367771:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5026 00000100:00080000:3.0:1775767565.367776:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767565.367781:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767565.367794:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767565.367795:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099072384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767565.367813:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099072512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767565.367828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767565.367833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767565.367835:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767565.367840:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5026 00000100:00080000:3.0:1775767565.367845:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767565.367849:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767565.367854:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767565.367867:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099072640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767565.367881:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767565.367891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767565.367895:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767565.367898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767565.367903:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5026 00000100:00080000:3.0:1775767565.367908:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767565.367913:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767565.367918:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767565.367932:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099072768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767565.367937:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767565/real 1775767565] req@ffff8b23864f0700 x1862022099072384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767565.367950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767565.367955:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767565.367957:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767565.367962:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5026 00000100:00080000:3.0:1775767565.367984:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767565.367990:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767565.367993:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099072384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775767565.367996:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775767565.368013:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1775767565.368013:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099072896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767565.368023:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767565.368032:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099072384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767565.368060:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099072512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767565.368090:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767565.368101:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767565/real 1775767565] req@ffff8b23864f1180 x1862022099072512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767565.368110:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099072512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767565.368121:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767565.368124:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767565.368126:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099072512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767565.368130:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099072640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767565.368141:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767565.368148:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767565/real 1775767565] req@ffff8b23864f3b80 x1862022099072640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767565.368155:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099072640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767565.368160:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767565.368162:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767565.368164:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099072640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767565.368167:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099072768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767565.368180:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767565.368185:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767565/real 1775767565] req@ffff8b23864f2d80 x1862022099072768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767565.368193:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099072768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767565.368198:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767565.368200:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767565.368202:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099072768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767565.368206:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099072896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767565.368220:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767565.368225:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767565/real 1775767565] req@ffff8b23864f3100 x1862022099072896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767565.368232:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099072896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767565.368238:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767565.368239:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767565.368241:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099072896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767566.263625:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767567.287619:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767568.311637:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767569.335670:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767570.359627:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767570.487531:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767570.487542:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767570.487547:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767570.487557:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5031 00000100:00080000:3.0:1775767570.487577:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767570.487585:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767570.487629:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099073024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767570.487654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767570.487659:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767570.487661:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767570.487665:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5031 00000100:00080000:3.0:1775767570.487668:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767570.487672:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767570.487684:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767570.487699:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099073152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767570.487708:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099073024:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767570.487713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767570.487716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767570.487718:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767570.487722:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5031 00000100:00080000:3.0:1775767570.487726:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767570.487729:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767570.487734:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767570.487744:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099073280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767570.487752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767570.487756:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767570.487758:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767570.487762:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5031 00000100:00080000:3.0:1775767570.487765:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767570.487768:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767570.487773:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767570.487782:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099073408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767570.487785:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767570.487791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767570.487794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767570.487795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767570.487799:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5031 00000100:00080000:3.0:1775767570.487802:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767570.487805:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767570.487809:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767570.487818:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099073536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767570.487829:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767570/real 1775767570] req@ffff8b23864f0000 x1862022099073024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767570.487850:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099073024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767570.487862:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767570.487867:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767570.487870:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099073024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767570.487901:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099073152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767570.487937:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767570.487947:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767570/real 1775767570] req@ffff8b23864f2a00 x1862022099073152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767570.487956:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099073152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767570.487961:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767570.487964:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767570.487966:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099073152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767570.487969:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099073280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767570.487982:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767570.487988:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767570/real 1775767570] req@ffff8b23864f3480 x1862022099073280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767570.487995:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099073280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767570.488000:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767570.488002:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767570.488003:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099073280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767570.488007:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099073408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767570.488019:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767570.488024:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767570/real 1775767570] req@ffff8b23864f3100 x1862022099073408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767570.488033:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099073408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767570.488038:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767570.488039:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767570.488041:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099073408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767570.488044:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099073536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767570.488055:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767570.488060:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767570/real 1775767570] req@ffff8b23864f2d80 x1862022099073536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767570.488066:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099073536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767570.488071:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767570.488074:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767570.488076:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099073536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767571.383683:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767572.407639:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767573.431701:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767574.455637:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767575.479656:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767575.607613:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767575.607624:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767575.607630:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767575.607639:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5037 00000100:00080000:2.0:1775767575.607652:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767575.607659:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767575.607705:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099073664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767575.607734:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767575.607737:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767575.607739:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767575.607742:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5037 00000100:00080000:2.0:1775767575.607744:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767575.607746:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767575.607755:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767575.607765:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099073792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767575.607775:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767575.607777:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767575.607779:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767575.607782:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5037 00000100:00080000:2.0:1775767575.607786:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767575.607794:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767575.607796:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099073664:192.168.202.128@tcp:250:kworker.0 00000008:00100000:2.0:1775767575.607799:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767575.607818:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099073920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767575.607826:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767575.607828:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767575.607830:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767575.607835:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5037 00000100:00080000:2.0:1775767575.607837:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767575.607840:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767575.607843:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767575.607853:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099074048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767575.607858:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767575.607860:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767575.607861:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767575.607863:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5037 00000100:00080000:2.0:1775767575.607881:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767575.607883:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767575.607892:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767575.607901:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099074176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767575.607908:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767575.607958:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767575/real 1775767575] req@ffff8b23b9ea4700 x1862022099073664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767575.607973:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099073664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767575.607982:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767575.607986:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767575.607989:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099073664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767575.607994:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099073792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767575.608012:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767575.608020:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767575/real 1775767575] req@ffff8b23b9ea5880 x1862022099073792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767575.608029:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099073792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767575.608034:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767575.608036:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767575.608038:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099073792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767575.608058:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099073920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767575.608078:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767575.608085:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767575/real 1775767575] req@ffff8b23b9ea4000 x1862022099073920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767575.608093:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099073920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767575.608098:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767575.608101:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767575.608103:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099073920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767575.608108:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099074048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767575.608120:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767575.608126:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767575/real 1775767575] req@ffff8b23b9ea6a00 x1862022099074048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767575.608133:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099074048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767575.608138:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767575.608140:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767575.608142:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099074048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767575.608146:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099074176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767575.608158:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767575.608163:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767575/real 1775767575] req@ffff8b23b9ea4a80 x1862022099074176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767575.608170:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099074176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767575.608175:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767575.608177:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767575.608179:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099074176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767576.503607:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767577.527663:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767578.551678:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767579.575681:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767580.599632:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767580.727596:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767580.727611:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767580.727617:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767580.727630:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5042 00000100:00080000:2.0:1775767580.727645:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767580.727654:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767580.727704:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099074304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767580.727742:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767580.727745:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767580.727747:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767580.727755:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5042 00000100:00080000:2.0:1775767580.727759:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767580.727762:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767580.727773:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767580.727784:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099074432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767580.727794:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767580.727797:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767580.727799:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767580.727802:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5042 00000100:00080000:2.0:1775767580.727804:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767580.727807:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767580.727812:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767580.727835:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099074304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775767580.727856:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099074560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767580.727865:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767580.727868:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767580.727869:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767580.727873:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5042 00000100:00080000:2.0:1775767580.727876:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767580.727879:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767580.727884:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767580.727892:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099074688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767580.727898:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767580.727901:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767580.727902:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767580.727905:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5042 00000100:00080000:2.0:1775767580.727907:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767580.727909:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767580.727912:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767580.727920:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099074816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767580.727970:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099074432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767580.728017:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099074560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767580.728046:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099074688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767580.728072:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099074816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767581.623674:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767582.647673:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767583.671646:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767583.799769:0:2364:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775767583.799847:0:2364:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767583.799870:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767583.799915:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767583.799923:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767583.799930:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767583.799938:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775767583.799989:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767580/real 1775767583] req@ffff8b23b9ea7b80 x1862022099074560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767583.800024:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099074560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767583.800039:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767583.800047:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767583.800053:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099074560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767583.800062:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767580/real 1775767583] req@ffff8b23b9ea6680 x1862022099074688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767583.800077:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099074688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767583.800085:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767583.800090:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767583.800094:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099074688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767583.800101:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767580/real 1775767583] req@ffff8b23b9ea5f80 x1862022099074816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767583.800129:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099074816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767583.800137:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767583.800140:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767583.800144:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099074816:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767583.800151:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767580/real 1775767583] req@ffff8b23b9ea6d80 x1862022099074304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767583.800164:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099074304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767583.800171:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767583.800175:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767583.800178:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099074304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767583.800185:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767580/real 1775767583] req@ffff8b23b9ea5c00 x1862022099074432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767583.800197:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099074432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767583.800204:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767583.800207:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767583.800209:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099074432:192.168.202.128@tcp:38:kworker.0 00010000:00010000:0.0:1775767584.695696:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767585.719630:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767585.847538:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767585.847550:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767585.847557:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767585.847569:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5047 00000100:00080000:2.0:1775767585.847584:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767585.847593:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767585.847639:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022099074944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767585.847668:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767585.847672:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767585.847673:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767585.847676:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5047 00000100:00080000:2.0:1775767585.847679:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767585.847682:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767585.847692:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767585.847707:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099075072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767585.847721:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767585.847725:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767585.847728:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767585.847733:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5047 00000100:00080000:2.0:1775767585.847738:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767585.847743:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767585.847751:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767585.847760:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099074944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775767585.847785:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099075200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767585.847811:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767585.847816:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767585.847819:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767585.847826:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5047 00000100:00080000:2.0:1775767585.847834:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767585.847850:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775767585.847856:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:2.0:1775767585.847859:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767585.847880:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099075328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767585.847893:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767585.847897:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767585.847900:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767585.847910:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5047 00000100:00100000:0.0:1775767585.847918:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767585/real 1775767585] req@ffff8b23b9ea5500 x1862022099074944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767585.847918:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767585.847929:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767585.847940:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767585.847971:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099075456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767585.847974:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022099074944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767585.848045:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767585.848051:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767585.848057:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099074944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767585.848075:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099075072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767585.848105:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767585.848120:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767585/real 1775767585] req@ffff8b23b9ea5c00 x1862022099075072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767585.848134:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099075072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767585.848142:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767585.848146:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767585.848150:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099075072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767585.848181:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099075200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767585.848212:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767585.848223:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767585/real 1775767585] req@ffff8b23b9ea6d80 x1862022099075200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767585.848241:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099075200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767585.848249:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767585.848252:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767585.848255:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099075200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767585.848261:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099075328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767585.848283:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767585.848308:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767585/real 1775767585] req@ffff8b23b9ea5f80 x1862022099075328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767585.848332:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099075328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767585.848341:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767585.848344:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767585.848347:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099075328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767585.848355:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099075456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767585.848380:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767585.848392:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767585/real 1775767585] req@ffff8b23b9ea6680 x1862022099075456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767585.848405:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099075456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767585.848420:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767585.848423:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767585.848427:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099075456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767586.743710:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767587.767605:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767588.791694:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767589.815592:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767590.839664:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767590.967554:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767590.967562:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767590.967566:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767590.967575:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5052 00000100:00080000:0.0:1775767590.967584:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767590.967590:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767590.967621:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099075584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767590.967645:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767590.967648:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767590.967650:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767590.967653:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5052 00000100:00080000:0.0:1775767590.967655:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767590.967658:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767590.967665:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767590.967674:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099075712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767590.967684:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099075584:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767590.967691:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767590.967693:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767590.967695:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767590.967698:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5052 00000100:00080000:0.0:1775767590.967700:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767590.967702:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767590.967705:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767590.967715:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099075840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767590.967734:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767590.967736:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767590.967737:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767590.967741:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5052 00000100:00080000:0.0:1775767590.967750:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767590.967753:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767590.967758:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767590.967759:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767590.967771:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099075968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767590.967778:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767590.967780:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767590.967782:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767590.967784:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5052 00000100:00080000:0.0:1775767590.967787:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767590.967790:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767590.967794:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767590.967799:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767590/real 1775767590] req@ffff8b2382dae680 x1862022099075584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767590.967805:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099076096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767590.967825:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099075584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767590.967838:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767590.967842:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767590.967845:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099075584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767590.967864:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099075712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767590.967880:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767590.967888:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767590/real 1775767590] req@ffff8b2382dad500 x1862022099075712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767590.967896:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099075712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767590.967901:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767590.967903:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767590.967905:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099075712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767590.967908:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099075840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767590.967919:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767590.967925:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767590/real 1775767590] req@ffff8b2382daea00 x1862022099075840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767590.967932:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099075840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767590.967937:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767590.967939:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767590.967941:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099075840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767590.967945:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099075968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767590.967956:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767590.967961:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767590/real 1775767590] req@ffff8b2382daed80 x1862022099075968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767590.967968:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099075968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767590.967974:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767590.967975:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767590.967977:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099075968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767590.967980:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099076096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767590.967990:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767590.967994:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767590/real 1775767590] req@ffff8b2382daf480 x1862022099076096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767590.968001:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099076096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767590.968006:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767590.968008:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767590.968010:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099076096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767591.863752:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767592.887687:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767593.911620:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767594.935675:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767595.959650:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767596.087726:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767596.087741:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767596.087748:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767596.087761:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5057 00000100:00080000:2.0:1775767596.087779:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767596.087788:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767596.087838:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099076224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767596.087868:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767596.087872:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767596.087878:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767596.087882:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5057 00000100:00080000:2.0:1775767596.087885:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767596.087888:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767596.087918:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767596.087934:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099076352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767596.087949:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767596.087954:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767596.087956:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767596.087984:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5057 00000100:00100000:3.0:1775767596.087987:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099076224:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767596.087988:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767596.087996:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767596.088002:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767596.088018:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099076480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767596.088030:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767596.088035:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767596.088037:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767596.088042:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5057 00000100:00080000:2.0:1775767596.088057:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767596.088062:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767596.088068:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767596.088069:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775767596.088084:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099076608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767596.088097:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767596.088100:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767596.088101:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767596.088105:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5057 00000100:00080000:2.0:1775767596.088108:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767596.088110:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767596.088113:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767596.088127:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099076736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767596.088131:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767596/real 1775767596] req@ffff8b23b9ea7b80 x1862022099076224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767596.088166:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099076224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767596.088176:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767596.088179:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767596.088184:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099076224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767596.088190:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099076352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767596.088207:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767596.088216:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767596/real 1775767596] req@ffff8b23b9ea4a80 x1862022099076352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767596.088225:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099076352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767596.088230:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767596.088232:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767596.088234:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099076352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767596.088257:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099076480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767596.088268:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767596.088275:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767596/real 1775767596] req@ffff8b23b9ea6a00 x1862022099076480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767596.088283:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099076480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767596.088293:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767596.088296:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767596.088298:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099076480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767596.088302:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099076608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767596.088312:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767596.088318:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767596/real 1775767596] req@ffff8b23b9ea4000 x1862022099076608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767596.088326:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099076608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767596.088331:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767596.088333:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767596.088335:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099076608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767596.088338:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099076736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767596.088349:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767596.088354:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767596/real 1775767596] req@ffff8b23b9ea5880 x1862022099076736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767596.088361:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099076736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767596.088366:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767596.088368:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767596.088371:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099076736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767596.983590:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767598.007709:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767599.031662:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767600.055661:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767601.079671:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767601.207595:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767601.207609:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767601.207615:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767601.207655:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5062 00000100:00080000:0.0:1775767601.207670:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767601.207678:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767601.207731:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022099076864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767601.207777:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767601.207781:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767601.207784:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767601.207788:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5062 00000100:00080000:0.0:1775767601.207792:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767601.207798:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767601.207810:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767601.207828:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022099076992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767601.207838:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099076864:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767601.207845:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767601.207850:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767601.207852:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767601.207858:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5062 00000100:00080000:0.0:1775767601.207864:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767601.207869:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767601.207876:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767601.207891:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022099077120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767601.207902:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767601.207906:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767601.207908:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767601.207913:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5062 00000800:00000100:3.0:1775767601.207917:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775767601.207921:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767601.207927:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767601.207946:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767601.207962:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022099077248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767601.207971:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767601/real 1775767601] req@ffff8b2382dafb80 x1862022099076864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767601.207976:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767601.207986:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767601.207990:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767601.207998:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5062 00000100:00080000:0.0:1775767601.208004:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767601.208006:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022099076864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767601.208012:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:3.0:1775767601.208021:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1775767601.208022:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775767601.208028:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767601.208032:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099076864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767601.208040:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099077376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767601.208062:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099076992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767601.208092:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767601.208103:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767601/real 1775767601] req@ffff8b2382dadf80 x1862022099076992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767601.208111:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022099076992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767601.208117:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767601.208119:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767601.208122:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099076992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767601.208126:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099077120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767601.208144:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767601.208152:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767601/real 1775767601] req@ffff8b2382dace00 x1862022099077120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767601.208160:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022099077120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767601.208166:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767601.208167:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767601.208169:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099077120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767601.208174:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099077248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767601.208186:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767601.208191:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767601/real 1775767601] req@ffff8b2382dac700 x1862022099077248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767601.208199:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022099077248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767601.208204:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767601.208206:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767601.208207:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099077248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767601.208211:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099077376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767601.208220:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767601.208225:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767601/real 1775767601] req@ffff8b2382daf480 x1862022099077376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767601.208232:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099077376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767601.208238:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767601.208240:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767601.208242:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099077376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767602.103604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767603.127656:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767604.151714:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767605.175676:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767606.199680:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767606.327539:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767606.327549:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767606.327553:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767606.327565:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5067 00000100:00080000:0.0:1775767606.327590:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767606.327597:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767606.327629:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099077504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767606.327654:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767606.327657:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767606.327658:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767606.327661:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5067 00000100:00080000:0.0:1775767606.327663:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767606.327666:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767606.327673:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767606.327682:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099077632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767606.327692:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767606.327696:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1775767606.327697:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099077504:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767606.327698:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767606.327706:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5067 00000100:00080000:0.0:1775767606.327708:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767606.327710:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767606.327713:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767606.327725:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099077760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767606.327731:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767606.327734:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767606.327735:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767606.327738:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5067 00000100:00080000:0.0:1775767606.327739:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767606.327742:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767606.327747:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767606.327758:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099077888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767606.327766:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767606.327771:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:3.0:1775767606.327774:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775767606.327775:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767606.327782:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5067 00000100:00080000:0.0:1775767606.327789:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767606.327793:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767606.327799:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767606.327813:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099078016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767606.327825:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767606/real 1775767606] req@ffff8b2382daed80 x1862022099077504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767606.327845:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099077504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767606.327853:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767606.327857:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767606.327860:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099077504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767606.327881:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099077632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767606.327900:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767606.327908:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767606/real 1775767606] req@ffff8b2382daea00 x1862022099077632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767606.327916:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099077632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767606.327922:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767606.327924:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767606.327927:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099077632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767606.327931:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099077760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767606.327942:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767606.327947:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767606/real 1775767606] req@ffff8b2382dad500 x1862022099077760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767606.327954:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099077760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767606.327959:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767606.327961:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767606.327963:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099077760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767606.327966:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099077888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767606.327978:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767606.327987:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767606/real 1775767606] req@ffff8b2382dae680 x1862022099077888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767606.328000:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099077888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767606.328007:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767606.328011:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767606.328014:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099077888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767606.328021:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099078016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767606.328042:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767606.328054:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767606/real 1775767606] req@ffff8b2382dac380 x1862022099078016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767606.328067:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099078016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767606.328075:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767606.328079:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767606.328082:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099078016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767607.223647:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767608.247704:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767609.271655:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767610.295617:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767611.319728:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767611.447646:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767611.447660:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767611.447667:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767611.447679:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5072 00000100:00080000:0.0:1775767611.447693:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767611.447699:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767611.447744:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022099078144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767611.447773:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767611.447776:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767611.447777:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767611.447781:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5072 00000100:00080000:0.0:1775767611.447783:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767611.447786:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767611.447795:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767611.447804:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022099078272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767611.447813:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767611.447821:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767611.447822:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767611.447826:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5072 00000100:00080000:0.0:1775767611.447828:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767611.447833:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767611.447836:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099078144:192.168.202.128@tcp:250:kworker.0 00000008:00100000:0.0:1775767611.447837:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767611.447850:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099078400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767611.447857:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767611.447859:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767611.447861:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767611.447863:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5072 00000100:00080000:0.0:1775767611.447865:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767611.447868:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767611.447871:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767611.447879:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022099078528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767611.447886:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767611.447889:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767611.447890:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767611.447893:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5072 00000100:00080000:0.0:1775767611.447896:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767611.447899:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767611.447902:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767611.447913:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099078656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767611.447929:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767611.447991:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767611/real 1775767611] req@ffff8b2382dac000 x1862022099078144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767611.448015:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022099078144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767611.448029:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767611.448034:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767611.448039:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099078144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767611.448048:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099078272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767611.448076:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767611.448091:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767611/real 1775767611] req@ffff8b2382daf100 x1862022099078272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767611.448105:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022099078272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767611.448122:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767611.448126:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767611.448130:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099078272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767611.448162:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099078400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767611.448186:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767611.448199:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767611/real 1775767611] req@ffff8b2382daca80 x1862022099078400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767611.448212:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099078400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767611.448222:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767611.448226:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767611.448230:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099078400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767611.448236:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099078528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767611.448258:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767611.448269:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767611/real 1775767611] req@ffff8b2382dad880 x1862022099078528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767611.448282:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022099078528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767611.448291:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767611.448294:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767611.448299:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099078528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767611.448306:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099078656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767611.448329:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767611.448341:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767611/real 1775767611] req@ffff8b2382dac380 x1862022099078656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767611.448370:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099078656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767611.448379:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767611.448383:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767611.448386:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099078656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767612.343711:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767613.367707:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767614.391720:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767615.415639:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767616.439740:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767616.567675:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767616.567689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767616.567694:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767616.567705:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5077 00000100:00080000:1.0:1775767616.567719:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767616.567726:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767616.567773:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099078784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767616.567810:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767616.567814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767616.567815:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767616.567818:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5077 00000100:00080000:1.0:1775767616.567820:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767616.567823:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767616.567833:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767616.567844:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099078912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767616.567854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767616.567857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767616.567858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767616.567861:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5077 00000100:00080000:1.0:1775767616.567864:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767616.567867:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767616.567870:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767616.567880:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099079040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767616.567882:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099078784:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767616.567896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767616.567900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767616.567902:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767616.567907:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5077 00000100:00080000:1.0:1775767616.567911:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767616.567915:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767616.567931:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767616.567946:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099079168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767616.567957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767616.567959:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767616.567961:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767616.567965:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5077 00000100:00080000:1.0:1775767616.567969:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767616.567972:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767616.567977:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767616.567983:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775767616.567990:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099079296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767616.568048:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767616/real 1775767616] req@ffff8b23864fc000 x1862022099078784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767616.568073:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099078784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767616.568083:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767616.568086:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767616.568089:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099078784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767616.568096:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099078912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767616.568116:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767616.568126:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767616/real 1775767616] req@ffff8b23864fed80 x1862022099078912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767616.568134:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099078912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767616.568140:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767616.568142:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767616.568145:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099078912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767616.568169:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099079040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767616.568189:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767616.568200:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767616/real 1775767616] req@ffff8b23864ff800 x1862022099079040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767616.568213:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099079040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767616.568222:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767616.568226:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767616.568229:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099079040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767616.568236:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099079168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767616.568257:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767616.568266:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767616/real 1775767616] req@ffff8b23864fdf80 x1862022099079168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767616.568278:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099079168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767616.568285:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767616.568287:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767616.568289:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099079168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767616.568293:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099079296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767616.568307:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767616.568313:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767616/real 1775767616] req@ffff8b23864fc380 x1862022099079296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767616.568320:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099079296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767616.568325:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767616.568327:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767616.568329:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099079296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767617.463656:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767618.487625:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767619.511613:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767620.535649:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767621.559626:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767621.687586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767621.687601:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767621.687606:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767621.687617:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5083 00000100:00080000:1.0:1775767621.687631:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767621.687638:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767621.687699:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099079424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767621.687726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767621.687730:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767621.687731:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767621.687734:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5083 00000100:00080000:1.0:1775767621.687736:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767621.687739:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767621.687760:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767621.687776:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099079552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767621.687787:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099079424:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767621.687792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767621.687797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767621.687800:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767621.687806:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5083 00000100:00080000:1.0:1775767621.687811:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767621.687816:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767621.687822:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767621.687837:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099079680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767621.687847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767621.687851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767621.687853:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767621.687858:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5083 00000100:00080000:1.0:1775767621.687872:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767621.687877:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767621.687883:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767621.687883:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775767621.687898:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099079808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767621.687906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767621.687909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767621.687910:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767621.687913:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5083 00000100:00080000:1.0:1775767621.687917:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767621.687920:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767621.687924:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767621.687934:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099079936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767621.687941:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767621/real 1775767621] req@ffff8b23864fce00 x1862022099079424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767621.687966:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099079424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767621.687978:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767621.687983:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767621.687988:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099079424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767621.688020:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099079552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767621.688045:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767621.688056:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767621/real 1775767621] req@ffff8b23864fd180 x1862022099079552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767621.688065:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099079552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767621.688071:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767621.688074:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767621.688076:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099079552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767621.688080:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099079680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767621.688093:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767621.688098:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767621/real 1775767621] req@ffff8b23864fea00 x1862022099079680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767621.688106:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099079680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767621.688112:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767621.688114:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767621.688116:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099079680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767621.688119:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099079808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767621.688131:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767621.688136:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767621/real 1775767621] req@ffff8b23864fca80 x1862022099079808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767621.688144:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099079808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767621.688149:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767621.688151:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767621.688153:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099079808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767621.688156:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099079936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767621.688166:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767621.688171:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767621/real 1775767621] req@ffff8b23864fe680 x1862022099079936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767621.688178:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099079936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767621.688183:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767621.688185:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767621.688187:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099079936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767622.583616:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767623.607607:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767624.631814:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767625.655669:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767626.679642:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767626.807620:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767626.807633:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767626.807640:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767626.807653:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5088 00000100:00080000:2.0:1775767626.807669:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767626.807677:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767626.807731:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099080064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767626.807769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767626.807779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767626.807781:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767626.807789:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5088 00000100:00080000:2.0:1775767626.807795:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767626.807800:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767626.807813:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767626.807817:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099080064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775767626.807832:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099080192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767626.807846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767626.807851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767626.807853:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767626.807858:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5088 00000100:00080000:2.0:1775767626.807890:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767626.807897:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767626.807903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767626.807914:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775767626.807919:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099080320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767626.807931:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767626.807936:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767626.807938:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767626.807944:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5088 00000100:00080000:2.0:1775767626.807949:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767626.807953:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767626.807959:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767626.807972:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099080448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767626.807981:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767626/real 1775767626] req@ffff8b23b9ea4700 x1862022099080064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767626.807987:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767626.807995:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767626.808000:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767626.808007:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5088 00000100:00080000:2.0:1775767626.808012:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767626.808015:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099080064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767626.808021:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767626.808031:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775767626.808047:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1775767626.808052:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099080576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767626.808057:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767626.808066:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099080064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767626.808095:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099080192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767626.808116:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767626.808126:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767626/real 1775767626] req@ffff8b23b9ea7480 x1862022099080192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767626.808134:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099080192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767626.808139:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767626.808142:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767626.808144:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099080192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767626.808148:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099080320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767626.808160:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767626.808170:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767626/real 1775767626] req@ffff8b23b9ea6300 x1862022099080320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767626.808182:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099080320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767626.808190:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767626.808193:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767626.808197:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099080320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767626.808205:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099080448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767626.808228:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767626.808240:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767626/real 1775767626] req@ffff8b23b9ea7100 x1862022099080448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767626.808253:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099080448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767626.808261:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767626.808264:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767626.808267:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099080448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767626.808273:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099080576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767626.808294:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767626.808305:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767626/real 1775767626] req@ffff8b23b9ea4380 x1862022099080576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767626.808318:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099080576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767626.808326:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767626.808329:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767626.808333:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099080576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767627.703707:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767628.727681:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767629.751702:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767630.775661:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767631.799775:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767631.927651:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767631.927662:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767631.927674:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767631.927688:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5093 00000100:00080000:1.0:1775767631.927702:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767631.927708:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767631.927759:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099080704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767631.927786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767631.927789:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767631.927790:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767631.927793:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5093 00000100:00080000:1.0:1775767631.927796:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767631.927798:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767631.927807:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767631.927818:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099080832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767631.927827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767631.927830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767631.927832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767631.927836:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5093 00000100:00080000:1.0:1775767631.927838:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767631.927841:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767631.927844:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767631.927859:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099080960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767631.927871:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099080704:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767631.927872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767631.927879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767631.927881:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767631.927891:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5093 00000100:00080000:1.0:1775767631.927893:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767631.927896:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767631.927898:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767631.927908:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099081088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767631.927915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767631.927917:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767631.927919:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767631.927922:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5093 00000100:00080000:1.0:1775767631.927924:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767631.927927:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767631.927931:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767631.927939:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099081216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767631.927963:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767631.928004:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767631/real 1775767631] req@ffff8b23864fdc00 x1862022099080704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767631.928019:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099080704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767631.928027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767631.928031:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767631.928034:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099080704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767631.928040:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099080832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767631.928056:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767631.928064:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767631/real 1775767631] req@ffff8b23864fd500 x1862022099080832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767631.928072:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099080832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767631.928078:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767631.928080:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767631.928082:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099080832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767631.928101:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099080960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767631.928113:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767631.928119:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767631/real 1775767631] req@ffff8b23864ff100 x1862022099080960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767631.928126:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099080960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767631.928136:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767631.928138:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767631.928140:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099080960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767631.928145:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099081088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767631.928155:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767631.928160:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767631/real 1775767631] req@ffff8b23864fe300 x1862022099081088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767631.928167:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099081088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767631.928172:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767631.928174:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767631.928176:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099081088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767631.928180:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099081216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767631.928189:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767631.928194:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767631/real 1775767631] req@ffff8b23864ffb80 x1862022099081216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767631.928201:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099081216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767631.928206:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767631.928208:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767631.928210:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099081216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767632.823626:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767633.847667:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767634.871624:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767635.895683:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767636.919649:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767637.047631:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767637.047643:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767637.047649:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767637.047663:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5098 00000100:00080000:3.0:1775767637.047678:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767637.047686:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767637.047741:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099081344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767637.047780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767637.047783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767637.047785:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767637.047788:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5098 00000100:00080000:3.0:1775767637.047792:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767637.047794:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767637.047804:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767637.047814:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099081472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767637.047823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767637.047825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767637.047826:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767637.047829:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5098 00000100:00080000:3.0:1775767637.047831:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767637.047834:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767637.047838:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767637.047847:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099081600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767637.047857:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767637.047861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767637.047863:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767637.047870:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5098 00000100:00100000:0.0:1775767637.047872:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099081344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767637.047875:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767637.047881:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767637.047888:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767637.047903:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099081728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767637.047914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767637.047918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767637.047920:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767637.047924:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5098 00000100:00080000:3.0:1775767637.047928:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767637.047932:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767637.047938:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767637.047951:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767637.047953:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099081856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767637.048000:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767637/real 1775767637] req@ffff8b23864f3b80 x1862022099081344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767637.048017:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099081344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767637.048026:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767637.048030:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767637.048033:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099081344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767637.048039:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099081472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767637.048056:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767637.048064:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767637/real 1775767637] req@ffff8b23864f1180 x1862022099081472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767637.048073:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099081472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767637.048078:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767637.048080:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767637.048082:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099081472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767637.048086:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099081600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767637.048097:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767637.048103:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767637/real 1775767637] req@ffff8b23864f0700 x1862022099081600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767637.048111:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099081600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767637.048116:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767637.048117:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767637.048119:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099081600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767637.048144:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099081728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767637.048156:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767637.048162:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767637/real 1775767637] req@ffff8b23864f2680 x1862022099081728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767637.048170:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099081728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767637.048175:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767637.048176:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767637.048178:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099081728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767637.048182:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099081856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767637.048192:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767637.048197:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767637/real 1775767637] req@ffff8b23864f0380 x1862022099081856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767637.048204:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099081856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767637.048210:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767637.048212:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767637.048213:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099081856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767637.943632:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767638.967627:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767639.991643:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767641.015586:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767642.039618:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767642.167566:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767642.167581:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767642.167587:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767642.167600:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5103 00000100:00080000:3.0:1775767642.167617:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767642.167624:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767642.167678:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099081984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767642.167714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767642.167719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767642.167721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767642.167726:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5103 00000100:00080000:3.0:1775767642.167730:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767642.167736:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767642.167749:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767642.167763:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099081984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767642.167770:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099082112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767642.167784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767642.167788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767642.167790:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767642.167795:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5103 00000100:00080000:3.0:1775767642.167800:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767642.167804:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767642.167810:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767642.167823:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099082240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767642.167834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767642.167838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767642.167840:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775767642.167846:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767642.167850:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5103 00000100:00080000:3.0:1775767642.167855:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767642.167860:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767642.167866:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767642.167880:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099082368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767642.167890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767642.167894:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767642/real 1775767642] req@ffff8b23864f1c00 x1862022099081984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767642.167897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767642.167901:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767642.167911:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5103 00000100:00080000:3.0:1775767642.167920:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767642.167926:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767642.167931:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099081984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775767642.167934:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775767642.167944:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767642.167949:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767642.167955:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099081984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767642.167956:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099082496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767642.167982:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099082112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767642.168000:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767642.168010:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767642/real 1775767642] req@ffff8b23864f1f80 x1862022099082112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767642.168020:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099082112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767642.168026:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767642.168028:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767642.168030:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099082112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767642.168034:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099082240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767642.168044:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767642.168049:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767642/real 1775767642] req@ffff8b23864f0a80 x1862022099082240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767642.168057:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099082240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767642.168067:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767642.168069:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767642.168071:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099082240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767642.168074:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099082368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767642.168086:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767642.168091:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767642/real 1775767642] req@ffff8b23864f0380 x1862022099082368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767642.168098:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099082368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767642.168103:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767642.168105:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767642.168106:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099082368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767642.168110:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099082496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767642.168123:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767642.168132:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767642/real 1775767642] req@ffff8b23864f2680 x1862022099082496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767642.168143:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099082496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767642.168152:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767642.168155:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767642.168159:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099082496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767643.063695:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767644.087604:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767645.111631:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767646.135689:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767647.159715:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767647.287543:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767647.287556:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767647.287562:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767647.287572:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5108 00000100:00080000:3.0:1775767647.287586:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767647.287593:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767647.287638:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099082624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767647.287668:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767647.287671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767647.287672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767647.287675:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5108 00000100:00080000:3.0:1775767647.287678:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767647.287680:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767647.287689:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767647.287700:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099082752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767647.287709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767647.287712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767647.287714:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767647.287721:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5108 00000100:00080000:3.0:1775767647.287723:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767647.287726:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767647.287729:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767647.287738:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099082880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767647.287746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767647.287754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1775767647.287756:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099082624:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767647.287756:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767647.287765:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5108 00000100:00080000:3.0:1775767647.287774:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767647.287777:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767647.287781:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767647.287789:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099083008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767647.287797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767647.287799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767647.287801:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767647.287804:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5108 00000100:00080000:3.0:1775767647.287806:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767647.287809:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767647.287814:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767647.287823:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099083136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767647.287875:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099082752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767647.287907:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099082880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767647.287932:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099083008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767647.287946:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099083136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775767648.183630:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767649.207701:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767650.231640:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775767650.359748:0:2365:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775767650.359784:0:2365:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775767650.359798:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767650.359827:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767650.359834:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767650.359839:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767650.359844:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775767650.359897:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767647/real 1775767650] req@ffff8b23864f3b80 x1862022099082880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767650.359927:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099082880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767650.359941:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767650.359947:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767650.359952:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099082880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767650.359967:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767647/real 1775767650] req@ffff8b23864f2d80 x1862022099083008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767650.359983:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099083008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767650.360000:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767650.360004:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767650.360008:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099083008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767650.360019:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767647/real 1775767650] req@ffff8b23864f3100 x1862022099083136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767650.360033:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099083136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767650.360042:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767650.360046:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767650.360049:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099083136:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767650.360057:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767647/real 1775767650] req@ffff8b23864f0700 x1862022099082624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767650.360070:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099082624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767650.360078:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767650.360082:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767650.360086:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099082624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767650.360094:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767647/real 1775767650] req@ffff8b23864f1180 x1862022099082752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767650.360105:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099082752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767650.360113:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767650.360117:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767650.360120:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099082752:192.168.202.128@tcp:38:kworker.0 00010000:00010000:1.0:1775767651.255707:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767652.279650:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767652.407639:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767652.407651:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767652.407656:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767652.407667:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5113 00000100:00080000:0.0:1775767652.407679:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767652.407686:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767652.407726:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099083264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767652.407757:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767652.407760:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767652.407762:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767652.407764:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5113 00000100:00080000:0.0:1775767652.407767:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767652.407770:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767652.407780:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767652.407789:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099083392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767652.407801:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767652.407807:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767652.407809:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767652.407810:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099083264:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767652.407816:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5113 00000100:00080000:0.0:1775767652.407823:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767652.407828:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767652.407834:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767652.407849:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099083520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767652.407859:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767652.407863:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767652.407865:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767652.407871:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5113 00000100:00080000:0.0:1775767652.407877:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767652.407882:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767652.407888:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767652.407903:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099083648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767652.407913:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767652.407917:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767652.407919:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767652.407926:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5113 00000800:00000100:3.0:1775767652.407929:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775767652.407933:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767652.407939:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767652.407946:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767652.407961:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099083776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767652.407996:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767652/real 1775767652] req@ffff8b2382dae680 x1862022099083264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767652.408019:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099083264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767652.408038:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767652.408043:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767652.408049:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099083264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767652.408086:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099083392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767652.408117:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767652.408151:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767652/real 1775767652] req@ffff8b2382dad500 x1862022099083392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767652.408168:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099083392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767652.408178:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767652.408183:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767652.408187:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099083392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767652.408194:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099083520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767652.408219:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767652.408232:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767652/real 1775767652] req@ffff8b2382daea00 x1862022099083520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767652.408246:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099083520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767652.408256:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767652.408260:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767652.408263:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099083520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767652.408271:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099083648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767652.408299:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767652.408311:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767652/real 1775767652] req@ffff8b2382daed80 x1862022099083648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767652.408324:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099083648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767652.408333:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767652.408337:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767652.408341:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099083648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767652.408347:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099083776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767652.408372:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767652.408385:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767652/real 1775767652] req@ffff8b2382daf480 x1862022099083776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767652.408399:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099083776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767652.408409:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767652.408413:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767652.408416:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099083776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767653.303617:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767654.327630:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767655.351654:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767656.375612:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767657.399703:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767657.527548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767657.527559:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767657.527563:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767657.527574:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5118 00000100:00080000:2.0:1775767657.527587:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767657.527595:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767657.527640:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022099083904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767657.527667:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767657.527670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767657.527672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767657.527675:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5118 00000100:00080000:2.0:1775767657.527677:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767657.527679:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767657.527689:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767657.527700:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099084032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767657.527709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767657.527712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767657.527713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767657.527716:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5118 00000100:00080000:2.0:1775767657.527719:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767657.527722:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767657.527724:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767657.527734:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099084160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767657.527734:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099083904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767657.527755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767657.527757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767657.527759:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767657.527762:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5118 00000100:00080000:2.0:1775767657.527763:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767657.527766:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767657.527769:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767657.527777:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099084288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767657.527782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767657.527784:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767657.527786:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767657.527788:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5118 00000100:00080000:2.0:1775767657.527791:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767657.527793:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767657.527796:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767657.527803:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099084416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767657.527828:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767657.527880:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767657/real 1775767657] req@ffff8b23b9ea4e00 x1862022099083904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767657.527903:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022099083904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767657.527916:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767657.527921:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767657.527926:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099083904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767657.527934:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099084032:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767657.527958:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767657.527971:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767657/real 1775767657] req@ffff8b23b9ea4380 x1862022099084032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767657.527985:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099084032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767657.527994:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767657.527997:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767657.528001:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099084032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767657.528030:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099084160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767657.528051:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767657.528061:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767657/real 1775767657] req@ffff8b23b9ea7100 x1862022099084160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767657.528091:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099084160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767657.528109:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767657.528113:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767657.528117:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099084160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767657.528131:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099084288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767657.528154:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767657.528165:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767657/real 1775767657] req@ffff8b23b9ea6300 x1862022099084288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767657.528179:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099084288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767657.528188:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767657.528201:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767657.528206:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099084288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767657.528213:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099084416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767657.528234:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767657.528245:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767657/real 1775767657] req@ffff8b23b9ea7480 x1862022099084416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767657.528258:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099084416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767657.528267:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767657.528271:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767657.528275:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099084416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767658.423603:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767659.447624:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767660.471605:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767661.495578:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767662.519660:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767662.647594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767662.647605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767662.647609:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767662.647619:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5124 00000100:00080000:2.0:1775767662.647634:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767662.647641:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767662.647687:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099084544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767662.647720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767662.647723:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767662.647725:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767662.647728:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5124 00000100:00080000:2.0:1775767662.647731:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767662.647733:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767662.647742:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767662.647752:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099084672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767662.647761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767662.647764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767662.647766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767662.647769:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5124 00000100:00080000:2.0:1775767662.647771:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767662.647776:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099084544:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767662.647777:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767662.647783:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767662.647791:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099084800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767662.647798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767662.647801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767662.647803:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767662.647806:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5124 00000100:00080000:2.0:1775767662.647808:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767662.647811:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767662.647814:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767662.647822:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099084928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767662.647828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767662.647830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767662.647831:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767662.647834:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5124 00000100:00080000:2.0:1775767662.647837:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767662.647839:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767662.647843:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767662.647850:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099085056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767662.647857:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767662.647912:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767662/real 1775767662] req@ffff8b23b9ea4700 x1862022099084544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767662.647929:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099084544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767662.647938:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767662.647941:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767662.647945:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099084544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767662.647954:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099084672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767662.647973:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767662.647985:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767662/real 1775767662] req@ffff8b23b9ea5880 x1862022099084672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767662.647994:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099084672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767662.647999:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767662.648002:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767662.648004:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099084672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767662.648042:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099084800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767662.648059:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767662.648067:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767662/real 1775767662] req@ffff8b23b9ea4000 x1862022099084800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767662.648075:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099084800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767662.648081:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767662.648083:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767662.648085:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099084800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767662.648089:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099084928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767662.648100:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767662.648106:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767662/real 1775767662] req@ffff8b23b9ea6a00 x1862022099084928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767662.648113:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099084928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767662.648118:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767662.648120:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767662.648122:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099084928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767662.648125:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099085056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767662.648135:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767662.648140:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767662/real 1775767662] req@ffff8b23b9ea4a80 x1862022099085056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767662.648148:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099085056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767662.648154:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767662.648155:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767662.648157:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099085056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767663.543614:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767664.567661:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767665.591700:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767666.615646:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767667.639714:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767667.767647:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767667.767660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767667.767666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767667.767679:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5129 00000100:00080000:3.0:1775767667.767694:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767667.767701:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767667.767754:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099085184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767667.767793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767667.767798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767667.767800:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767667.767805:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5129 00000100:00080000:3.0:1775767667.767809:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767667.767814:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767667.767826:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767667.767845:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099085312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767667.767855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767667.767858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767667.767867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767667.767872:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5129 00000100:00080000:3.0:1775767667.767878:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767667.767879:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099085184:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767667.767884:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767667.767893:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767667.767902:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099085440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767667.767912:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767667.767916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767667.767918:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767667.767921:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5129 00000100:00080000:3.0:1775767667.767926:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767667.767930:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767667.767935:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767667.767950:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099085568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767667.767964:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767667.767968:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767667.767973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767667.767990:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767667.767997:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5129 00000100:00080000:3.0:1775767667.768002:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767667.768007:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767667.768013:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767667.768030:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099085696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767667.768039:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767667/real 1775767667] req@ffff8b23864f3480 x1862022099085184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767667.768076:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099085184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767667.768085:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767667.768090:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767667.768094:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099085184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767667.768101:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099085312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767667.768123:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767667.768132:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767667/real 1775767667] req@ffff8b23864f2a00 x1862022099085312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767667.768140:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099085312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767667.768145:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767667.768147:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767667.768149:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099085312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767667.768170:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099085440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767667.768182:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767667.768188:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767667/real 1775767667] req@ffff8b23864f0000 x1862022099085440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767667.768195:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099085440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767667.768200:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767667.768202:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767667.768204:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099085440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767667.768208:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099085568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767667.768219:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767667.768224:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767667/real 1775767667] req@ffff8b23864f1180 x1862022099085568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767667.768232:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099085568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767667.768237:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767667.768239:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767667.768240:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099085568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767667.768244:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099085696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767667.768254:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767667.768260:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767667/real 1775767667] req@ffff8b23864f0700 x1862022099085696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767667.768267:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099085696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767667.768272:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767667.768274:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767667.768276:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099085696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767668.663697:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767669.687670:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767670.711657:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767671.735601:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767672.759590:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767672.887541:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767672.887549:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767672.887553:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767672.887562:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5134 00000100:00080000:2.0:1775767672.887571:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767672.887577:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767672.887611:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099085824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767672.887633:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767672.887636:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767672.887637:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767672.887640:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5134 00000100:00080000:2.0:1775767672.887642:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767672.887645:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767672.887652:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767672.887660:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099085952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767672.887669:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767672.887673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775767672.887674:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099085824:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767672.887675:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767672.887682:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5134 00000100:00080000:2.0:1775767672.887685:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767672.887687:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767672.887689:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767672.887697:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099086080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767672.887704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767672.887707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767672.887708:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767672.887711:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5134 00000100:00080000:2.0:1775767672.887714:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767672.887717:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767672.887720:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767672.887728:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099086208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767672.887735:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775767672.887741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767672.887743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767672.887745:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767672.887749:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5134 00000100:00080000:2.0:1775767672.887752:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767672.887754:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767672.887758:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767672.887764:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099086336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767672.887778:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767672/real 1775767672] req@ffff8b23b9ea7b80 x1862022099085824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767672.887806:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099085824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767672.887818:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767672.887823:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767672.887827:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099085824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767672.887852:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099085952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767672.887898:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767672.887910:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767672/real 1775767672] req@ffff8b23b9ea6680 x1862022099085952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767672.887920:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099085952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767672.887963:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767672.887966:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767672.887968:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099085952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767672.887972:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099086080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767672.887989:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767672.887996:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767672/real 1775767672] req@ffff8b23b9ea5f80 x1862022099086080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767672.888003:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099086080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767672.888009:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767672.888026:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767672.888029:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099086080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767672.888033:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099086208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767672.888046:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767672.888052:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767672/real 1775767672] req@ffff8b23b9ea6d80 x1862022099086208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767672.888061:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099086208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767672.888066:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767672.888068:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767672.888071:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099086208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767672.888075:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099086336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767672.888086:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767672.888092:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767672/real 1775767672] req@ffff8b23b9ea5c00 x1862022099086336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767672.888102:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099086336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767672.888111:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767672.888131:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767672.888134:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099086336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767673.783642:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767674.807666:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767675.831665:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767676.855678:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767677.879645:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767678.007570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767678.007577:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767678.007581:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767678.007590:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5139 00000100:00080000:2.0:1775767678.007600:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767678.007607:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767678.007645:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022099086464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767678.007666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767678.007672:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767678.007673:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767678.007677:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5139 00000100:00080000:2.0:1775767678.007681:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767678.007684:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767678.007694:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767678.007705:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099086592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767678.007710:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099086464:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767678.007734:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767678.007737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767678.007738:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767678.007743:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5139 00000100:00080000:2.0:1775767678.007746:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767678.007749:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767678.007754:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767678.007763:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099086720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767678.007769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767678.007772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767678.007774:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767678.007777:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5139 00000100:00080000:2.0:1775767678.007781:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767678.007783:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767678.007787:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767678.007793:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099086848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767678.007800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767678.007802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767678.007804:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767678.007806:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5139 00000100:00080000:2.0:1775767678.007809:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775767678.007814:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775767678.007820:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767678.007825:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767678.007834:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099086976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767678.007868:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767678/real 1775767678] req@ffff8b23b9ea5500 x1862022099086464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767678.007910:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022099086464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767678.007919:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767678.007923:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767678.007927:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099086464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767678.007951:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099086592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767678.007970:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767678.007987:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767678/real 1775767678] req@ffff8b23b9ea5c00 x1862022099086592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767678.007996:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099086592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767678.008002:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767678.008004:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767678.008006:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099086592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767678.008010:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099086720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767678.008023:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767678.008029:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767678/real 1775767678] req@ffff8b23b9ea6d80 x1862022099086720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767678.008037:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099086720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767678.008042:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767678.008044:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767678.008045:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099086720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767678.008049:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099086848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767678.008063:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767678.008068:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767678/real 1775767678] req@ffff8b23b9ea5f80 x1862022099086848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767678.008076:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099086848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767678.008081:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767678.008084:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767678.008086:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099086848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767678.008090:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099086976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767678.008099:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767678.008105:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767678/real 1775767678] req@ffff8b23b9ea6680 x1862022099086976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767678.008111:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099086976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767678.008117:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767678.008119:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767678.008121:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099086976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767678.903623:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767679.927682:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767680.951590:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767681.975672:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767682.999606:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767683.127557:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767683.127566:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767683.127570:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767683.127579:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5144 00000100:00080000:2.0:1775767683.127588:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767683.127593:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767683.127628:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099087104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767683.127653:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767683.127657:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767683.127658:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767683.127660:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5144 00000100:00080000:2.0:1775767683.127663:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767683.127666:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767683.127673:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767683.127681:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099087232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767683.127689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767683.127692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767683.127693:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767683.127697:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5144 00000100:00080000:2.0:1775767683.127699:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767683.127702:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767683.127705:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767683.127718:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099087104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775767683.127719:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099087360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767683.127731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767683.127733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767683.127735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767683.127737:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5144 00000100:00080000:2.0:1775767683.127740:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767683.127743:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767683.127746:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767683.127754:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099087488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767683.127765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767683.127767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767683.127768:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767683.127770:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5144 00000100:00080000:2.0:1775767683.127773:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767683.127775:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767683.127778:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767683.127784:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775767683.127787:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099087616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767683.127822:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767683/real 1775767683] req@ffff8b23b9ea7b80 x1862022099087104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767683.127868:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099087104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767683.127878:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767683.127881:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767683.127885:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099087104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767683.127891:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099087232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767683.127909:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767683.127917:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767683/real 1775767683] req@ffff8b23b9ea4a80 x1862022099087232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767683.127925:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099087232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767683.127931:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767683.127933:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767683.127935:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099087232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767683.127952:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099087360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767683.127962:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767683.127968:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767683/real 1775767683] req@ffff8b23b9ea6a00 x1862022099087360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767683.127975:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099087360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767683.127980:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767683.127982:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767683.127983:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099087360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767683.127987:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099087488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767683.127997:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767683.128002:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767683/real 1775767683] req@ffff8b23b9ea4000 x1862022099087488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767683.128010:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099087488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767683.128014:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767683.128016:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767683.128018:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099087488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767683.128021:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099087616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767683.128031:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767683.128036:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767683/real 1775767683] req@ffff8b23b9ea5880 x1862022099087616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767683.128043:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099087616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767683.128048:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767683.128049:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767683.128051:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099087616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767684.023649:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767685.047596:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767686.071648:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767687.095619:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767688.119658:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767688.247602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767688.247613:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767688.247618:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767688.247628:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5149 00000100:00080000:3.0:1775767688.247641:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767688.247647:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767688.247694:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099087744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767688.247722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767688.247726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767688.247728:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767688.247731:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5149 00000100:00080000:3.0:1775767688.247733:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767688.247736:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767688.247753:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767688.247764:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099087872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767688.247773:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767688.247792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767688.247794:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767688.247798:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5149 00000100:00080000:3.0:1775767688.247801:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767688.247804:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767688.247808:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767688.247819:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099088000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767688.247826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767688.247829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767688.247830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767688.247833:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5149 00000100:00080000:3.0:1775767688.247835:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767688.247837:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767688.247840:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767688.247847:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099088128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767688.247856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767688.247859:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1775767688.247862:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099087744:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767688.247862:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767688.247868:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5149 00000100:00080000:3.0:1775767688.247873:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767688.247882:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767688.247886:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767688.247894:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099088256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767688.247947:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767688.247986:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767688/real 1775767688] req@ffff8b23864f3100 x1862022099087744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767688.248001:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099087744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767688.248015:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767688.248018:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767688.248021:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099087744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767688.248027:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099087872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767688.248045:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767688.248057:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767688/real 1775767688] req@ffff8b23864f2d80 x1862022099087872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767688.248070:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099087872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767688.248078:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767688.248082:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767688.248085:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099087872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767688.248091:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099088000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767688.248113:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767688.248123:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767688/real 1775767688] req@ffff8b23864f3b80 x1862022099088000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767688.248131:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099088000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767688.248138:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767688.248140:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767688.248142:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099088000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767688.248165:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099088128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767688.248178:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767688.248184:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767688/real 1775767688] req@ffff8b23864f2680 x1862022099088128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767688.248192:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099088128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767688.248205:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767688.248207:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767688.248210:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099088128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767688.248230:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099088256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767688.248246:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767688.248253:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767688/real 1775767688] req@ffff8b23864f0380 x1862022099088256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767688.248262:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099088256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767688.248268:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767688.248270:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767688.248272:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099088256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767689.143572:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767690.167582:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767691.191637:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767692.215676:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767693.239639:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767693.367620:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767693.367631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767693.367637:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767693.367647:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5154 00000100:00080000:0.0:1775767693.367660:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767693.367666:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767693.367712:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022099088384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767693.367751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767693.367755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767693.367756:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767693.367760:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5154 00000100:00080000:0.0:1775767693.367763:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767693.367766:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767693.367776:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767693.367788:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022099088512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767693.367798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775767693.367798:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099088384:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767693.367805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767693.367808:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767693.367812:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5154 00000100:00080000:0.0:1775767693.367814:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767693.367817:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767693.367820:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767693.367832:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022099088640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767693.367844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767693.367846:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767693.367848:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767693.367851:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5154 00000100:00080000:0.0:1775767693.367853:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767693.367855:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767693.367858:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767693.367866:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022099088768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767693.367873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767693.367875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767693.367876:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767693.367878:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5154 00000100:00080000:0.0:1775767693.367880:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767693.367882:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767693.367885:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767693.367891:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767693.367897:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099088896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767693.367943:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767693/real 1775767693] req@ffff8b2382dac700 x1862022099088384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767693.367960:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022099088384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767693.367969:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767693.367973:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767693.367976:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099088384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767693.368003:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099088512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767693.368021:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767693.368030:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767693/real 1775767693] req@ffff8b2382dace00 x1862022099088512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767693.368039:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022099088512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767693.368044:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767693.368047:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767693.368049:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099088512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767693.368053:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099088640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767693.368064:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767693.368069:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767693/real 1775767693] req@ffff8b2382dadf80 x1862022099088640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767693.368077:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022099088640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767693.368082:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767693.368084:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767693.368086:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099088640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767693.368089:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099088768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767693.368101:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767693.368107:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767693/real 1775767693] req@ffff8b2382dafb80 x1862022099088768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767693.368115:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022099088768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767693.368120:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767693.368121:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767693.368123:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099088768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767693.368126:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099088896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767693.368136:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767693.368142:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767693/real 1775767693] req@ffff8b2382daf480 x1862022099088896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767693.368149:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099088896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767693.368156:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767693.368159:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767693.368162:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099088896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767694.263758:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767695.287591:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767696.311663:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767697.335628:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767698.359727:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767698.487579:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767698.487592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767698.487599:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767698.487611:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5159 00000100:00080000:1.0:1775767698.487625:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767698.487632:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767698.487727:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099089024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767698.487758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767698.487762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767698.487764:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767698.487770:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5159 00000100:00080000:1.0:1775767698.487774:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767698.487778:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767698.487788:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767698.487795:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099089024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775767698.487804:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099089152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767698.487820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767698.487822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767698.487823:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767698.487827:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5159 00000100:00080000:1.0:1775767698.487830:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767698.487833:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767698.487837:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767698.487846:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099089280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767698.487854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767698.487857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767698.487858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767698.487861:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5159 00000100:00080000:1.0:1775767698.487863:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767698.487866:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767698.487870:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767698.487870:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775767698.487886:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099089408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767698.487893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767698.487896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767698.487897:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767698.487905:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5159 00000100:00080000:1.0:1775767698.487908:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767698.487911:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767698.487915:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767698.487922:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767698/real 1775767698] req@ffff8b23864ffb80 x1862022099089024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767698.487927:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099089536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767698.487958:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099089024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767698.487969:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767698.487973:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767698.487977:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099089024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767698.488000:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099089152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767698.488016:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767698.488025:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767698/real 1775767698] req@ffff8b23864fe300 x1862022099089152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767698.488033:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099089152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767698.488038:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767698.488040:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767698.488042:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099089152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767698.488046:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099089280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767698.488057:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767698.488062:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767698/real 1775767698] req@ffff8b23864ff100 x1862022099089280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767698.488069:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099089280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767698.488075:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767698.488077:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767698.488079:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099089280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767698.488083:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099089408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767698.488094:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767698.488100:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767698/real 1775767698] req@ffff8b23864fd500 x1862022099089408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767698.488107:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099089408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767698.488113:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767698.488114:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767698.488116:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099089408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767698.488119:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099089536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767698.488129:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767698.488134:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767698/real 1775767698] req@ffff8b23864fdc00 x1862022099089536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767698.488141:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099089536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767698.488146:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767698.488148:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767698.488151:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099089536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767699.383575:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767700.407701:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767701.431631:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767702.455696:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767703.479677:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767703.607625:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767703.607656:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767703.607662:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767703.607675:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5165 00000100:00080000:3.0:1775767703.607692:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767703.607701:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767703.607751:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099089664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767703.607785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767703.607788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767703.607790:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767703.607793:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5165 00000100:00080000:3.0:1775767703.607797:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767703.607800:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767703.607809:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767703.607818:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099089792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767703.607827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767703.607831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767703.607832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767703.607836:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5165 00000100:00080000:3.0:1775767703.607838:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767703.607841:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767703.607844:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767703.607853:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099089920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767703.607854:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099089664:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767703.607869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767703.607872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767703.607873:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767703.607876:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5165 00000100:00080000:3.0:1775767703.607878:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767703.607880:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767703.607883:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767703.607892:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099090048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767703.607898:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767703.607901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767703.607902:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767703.607905:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5165 00000100:00080000:3.0:1775767703.607907:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767703.607909:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767703.607912:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767703.607918:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099090176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767703.607942:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767703.607997:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767703/real 1775767703] req@ffff8b23864f0a80 x1862022099089664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767703.608021:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099089664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767703.608030:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767703.608033:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767703.608036:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099089664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767703.608042:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099089792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767703.608073:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767703.608083:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767703/real 1775767703] req@ffff8b23864f1f80 x1862022099089792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767703.608091:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099089792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767703.608097:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767703.608099:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767703.608101:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099089792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767703.608125:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099089920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767703.608137:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767703.608143:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767703/real 1775767703] req@ffff8b23864f1c00 x1862022099089920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767703.608151:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099089920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767703.608161:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767703.608163:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767703.608166:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099089920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767703.608169:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099090048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767703.608180:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767703.608185:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767703/real 1775767703] req@ffff8b23864f0380 x1862022099090048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767703.608192:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099090048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767703.608197:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767703.608199:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767703.608200:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099090048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767703.608204:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099090176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767703.608216:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767703.608222:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767703/real 1775767703] req@ffff8b23864f2680 x1862022099090176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767703.608229:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099090176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767703.608234:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767703.608236:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767703.608239:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099090176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767704.503587:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767705.527603:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767706.551709:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767707.575623:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767708.599717:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767708.727636:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767708.727644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767708.727648:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767708.727656:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5170 00000100:00080000:1.0:1775767708.727666:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767708.727671:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767708.727709:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099090304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767708.727746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767708.727749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767708.727750:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767708.727754:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5170 00000100:00080000:1.0:1775767708.727757:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767708.727760:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767708.727767:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767708.727778:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099090432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767708.727786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767708.727789:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767708.727790:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767708.727794:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5170 00000100:00080000:1.0:1775767708.727796:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767708.727799:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767708.727802:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767708.727806:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099090304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775767708.727813:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099090560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767708.727820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767708.727823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767708.727824:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767708.727827:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5170 00000100:00080000:1.0:1775767708.727828:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767708.727830:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767708.727833:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767708.727843:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099090688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767708.727849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767708.727852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767708.727853:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767708.727857:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5170 00000100:00080000:1.0:1775767708.727859:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775767708.727863:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775767708.727864:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767708.727870:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767708.727878:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099090816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767708.727911:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767708/real 1775767708] req@ffff8b23864fe680 x1862022099090304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767708.727927:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099090304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767708.727934:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767708.727938:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767708.727941:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099090304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767708.727946:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099090432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767708.727961:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767708.727969:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767708/real 1775767708] req@ffff8b23864fca80 x1862022099090432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767708.727976:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099090432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767708.727981:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767708.727983:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767708.727985:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099090432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767708.728005:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099090560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767708.728016:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767708.728022:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767708/real 1775767708] req@ffff8b23864fea00 x1862022099090560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767708.728030:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099090560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767708.728034:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767708.728036:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767708.728038:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099090560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767708.728041:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099090688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767708.728051:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767708.728057:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767708/real 1775767708] req@ffff8b23864fd180 x1862022099090688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767708.728064:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099090688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767708.728069:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767708.728070:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767708.728072:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099090688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767708.728075:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099090816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767708.728084:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767708.728088:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767708/real 1775767708] req@ffff8b23864fce00 x1862022099090816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767708.728095:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099090816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767708.728100:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767708.728102:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767708.728104:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099090816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767709.623702:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767710.647734:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767711.671649:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767712.695635:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767713.719682:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767713.847687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767713.847697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767713.847702:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767713.847713:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5175 00000100:00080000:3.0:1775767713.847725:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767713.847733:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767713.847788:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099090944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767713.847823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767713.847827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767713.847828:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767713.847832:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5175 00000100:00080000:3.0:1775767713.847835:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767713.847838:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767713.847849:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767713.847863:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099091072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767713.847872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767713.847876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767713.847877:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767713.847883:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099090944:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767713.847884:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5175 00000100:00080000:3.0:1775767713.847889:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767713.847893:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767713.847897:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767713.847907:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099091200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767713.847914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767713.847917:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767713.847918:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767713.847921:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5175 00000100:00080000:3.0:1775767713.847923:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767713.847925:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767713.847928:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767713.847935:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099091328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767713.847941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767713.847944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767713.847945:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767713.847947:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5175 00000100:00080000:3.0:1775767713.847950:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767713.847953:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767713.847955:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767713.847962:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099091456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767713.848035:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099091072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767713.848073:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099091200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767713.848100:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099091328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767713.848122:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099091456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767714.743645:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767715.767664:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767716.791632:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775767716.919788:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775767716.919837:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775767716.919855:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767716.919900:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767716.919909:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767716.919916:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767716.919925:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775767716.919979:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767713/real 1775767716] req@ffff8b23864f2d80 x1862022099091072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767716.920014:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099091072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767716.920028:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767716.920033:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767716.920039:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099091072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767716.920049:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767713/real 1775767716] req@ffff8b23864f3100 x1862022099091200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767716.920064:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099091200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767716.920080:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767716.920084:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767716.920086:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099091200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767716.920092:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767713/real 1775767716] req@ffff8b23864f0700 x1862022099091328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767716.920102:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099091328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767716.920111:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767716.920115:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767716.920118:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099091328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767716.920125:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767713/real 1775767716] req@ffff8b23864f1180 x1862022099091456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767716.920137:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099091456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767716.920147:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767716.920150:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767716.920154:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099091456:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767716.920162:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767713/real 1775767716] req@ffff8b23864f3b80 x1862022099090944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767716.920177:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099090944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767716.920193:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767716.920198:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767716.920202:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099090944:192.168.202.128@tcp:250:kworker.0 00010000:00010000:3.0:1775767717.815620:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767718.839705:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767718.967583:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767718.967594:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767718.967600:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767718.967611:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5180 00000100:00080000:3.0:1775767718.967623:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767718.967632:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767718.967674:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099091584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767718.967704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767718.967708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767718.967709:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767718.967713:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5180 00000100:00080000:3.0:1775767718.967715:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767718.967718:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767718.967728:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767718.967739:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099091712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767718.967751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767718.967755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767718.967757:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767718.967762:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5180 00000100:00080000:3.0:1775767718.967766:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767718.967772:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767718.967777:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099091584:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775767718.967779:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767718.967796:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099091840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767718.967816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767718.967820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767718.967823:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767718.967828:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5180 00000100:00080000:3.0:1775767718.967833:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767718.967837:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767718.967843:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767718.967857:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099091968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767718.967866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767718.967870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767718.967873:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767718.967877:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5180 00000800:00000100:0.0:1775767718.967882:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767718.967885:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767718.967892:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767718.967898:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767718.967912:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099092096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767718.967944:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767718/real 1775767718] req@ffff8b23864f0000 x1862022099091584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767718.967984:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099091584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767718.967998:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767718.968003:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767718.968009:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099091584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767718.968018:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099091712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767718.968050:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767718.968065:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767718/real 1775767718] req@ffff8b23864f2a00 x1862022099091712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767718.968079:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099091712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767718.968089:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767718.968094:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767718.968098:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099091712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767718.968143:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099091840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767718.968169:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767718.968182:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767718/real 1775767718] req@ffff8b23864f3480 x1862022099091840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767718.968196:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099091840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767718.968206:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767718.968211:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767718.968215:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099091840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767718.968222:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099091968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767718.968247:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767718.968259:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767718/real 1775767718] req@ffff8b23864f3b80 x1862022099091968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767718.968272:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099091968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767718.968282:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767718.968286:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767718.968290:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099091968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767718.968297:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099092096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767718.968319:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767718.968330:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767718/real 1775767718] req@ffff8b23864f1180 x1862022099092096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767718.968344:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099092096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767718.968354:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767718.968358:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767718.968362:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099092096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767719.863607:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767720.887721:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767721.911593:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767722.935702:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767723.959680:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767724.087617:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767724.087628:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767724.087632:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767724.087643:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5185 00000100:00080000:0.0:1775767724.087655:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767724.087662:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767724.087710:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099092224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767724.087744:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767724.087747:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767724.087748:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767724.087751:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5185 00000100:00080000:0.0:1775767724.087753:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767724.087756:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767724.087765:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767724.087775:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099092352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767724.087795:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767724.087798:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767724.087800:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767724.087807:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5185 00000100:00100000:3.0:1775767724.087807:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099092224:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767724.087809:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767724.087816:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767724.087823:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767724.087837:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099092480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767724.087843:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767724.087847:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767724.087848:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767724.087852:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5185 00000100:00080000:0.0:1775767724.087856:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767724.087858:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767724.087862:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767724.087871:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099092608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767724.087877:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767724.087879:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767724.087880:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767724.087883:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5185 00000100:00080000:0.0:1775767724.087886:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767724.087889:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767724.087892:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767724.087902:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099092736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767724.087914:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767724.087967:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767724/real 1775767724] req@ffff8b2382daed80 x1862022099092224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767724.087987:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099092224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767724.087999:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767724.088004:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767724.088008:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099092224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767724.088036:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099092352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767724.088064:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767724.088079:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767724/real 1775767724] req@ffff8b2382daea00 x1862022099092352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767724.088092:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099092352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767724.088101:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767724.088115:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767724.088120:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099092352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767724.088128:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099092480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767724.088152:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767724.088164:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767724/real 1775767724] req@ffff8b2382dad500 x1862022099092480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767724.088178:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099092480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767724.088184:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767724.088186:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767724.088189:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099092480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767724.088193:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099092608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767724.088214:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767724.088224:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767724/real 1775767724] req@ffff8b2382dae680 x1862022099092608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767724.088237:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099092608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767724.088245:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767724.088248:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767724.088252:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099092608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767724.088258:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099092736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767724.088280:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767724.088290:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767724/real 1775767724] req@ffff8b2382dac380 x1862022099092736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767724.088298:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099092736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767724.088310:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767724.088314:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767724.088317:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099092736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767724.983679:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767726.007619:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767727.031722:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767728.055689:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767729.079619:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767729.207566:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767729.207576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767729.207580:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767729.207592:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5190 00000100:00080000:3.0:1775767729.207602:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767729.207610:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767729.207647:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099092864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767729.207671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767729.207676:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767729.207678:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767729.207682:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5190 00000100:00080000:3.0:1775767729.207686:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767729.207690:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767729.207699:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767729.207705:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099092864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767729.207719:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099092992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767729.207730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767729.207734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767729.207736:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767729.207739:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5190 00000100:00080000:3.0:1775767729.207742:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767729.207746:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767729.207750:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767729.207759:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099093120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767729.207765:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767729.207768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767729.207771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767729.207773:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767729.207776:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5190 00000100:00080000:3.0:1775767729.207779:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767729.207782:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767729.207786:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767729.207795:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099093248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767729.207802:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767729/real 1775767729] req@ffff8b23864f0700 x1862022099092864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767729.207804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767729.207807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767729.207809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767729.207812:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5190 00000100:00080000:3.0:1775767729.207816:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767729.207819:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767729.207820:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099092864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767729.207830:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767729.207834:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000008:00100000:3.0:1775767729.207834:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767729.207839:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099092864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767729.207844:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099093376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767729.207862:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099092992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767729.207886:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767729.207899:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767729/real 1775767729] req@ffff8b23864f3100 x1862022099092992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767729.207907:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099092992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767729.207913:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767729.207915:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767729.207918:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099092992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767729.207922:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099093120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767729.207936:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767729.207942:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767729/real 1775767729] req@ffff8b23864f2d80 x1862022099093120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767729.207949:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099093120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767729.207954:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767729.207956:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767729.207958:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099093120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767729.207962:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099093248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767729.207973:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767729.207978:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767729/real 1775767729] req@ffff8b23864f2680 x1862022099093248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767729.207985:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099093248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767729.207990:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767729.207991:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767729.207993:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099093248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767729.207996:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099093376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767729.208006:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767729.208012:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767729/real 1775767729] req@ffff8b23864f0380 x1862022099093376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767729.208018:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099093376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767729.208023:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767729.208025:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767729.208027:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099093376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767730.103623:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767731.127686:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767732.151610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767733.175696:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767734.199638:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767734.327590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767734.327601:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767734.327607:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767734.327619:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5195 00000100:00080000:3.0:1775767734.327632:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767734.327639:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767734.327690:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099093504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767734.327728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767734.327732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767734.327734:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767734.327738:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5195 00000100:00080000:3.0:1775767734.327743:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767734.327747:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767734.327760:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767734.327776:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099093632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767734.327790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767734.327795:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775767734.327797:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099093504:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767734.327797:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767734.327803:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5195 00000100:00080000:3.0:1775767734.327806:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767734.327810:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767734.327815:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767734.327827:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099093760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767734.327835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767734.327837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767734.327839:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767734.327841:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5195 00000100:00080000:3.0:1775767734.327844:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767734.327847:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767734.327849:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767734.327856:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099093888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767734.327863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767734.327866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767734.327867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767734.327869:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5195 00000100:00080000:3.0:1775767734.327871:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767734.327876:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775767734.327877:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:3.0:1775767734.327881:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767734.327890:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099094016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767734.327926:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767734/real 1775767734] req@ffff8b23864f1c00 x1862022099093504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767734.327950:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099093504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767734.327971:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767734.327976:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767734.327980:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099093504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767734.328004:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099093632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767734.328024:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767734.328034:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767734/real 1775767734] req@ffff8b23864f1f80 x1862022099093632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767734.328043:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099093632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767734.328054:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767734.328056:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767734.328058:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099093632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767734.328062:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099093760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767734.328075:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767734.328081:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767734/real 1775767734] req@ffff8b23864f0a80 x1862022099093760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767734.328088:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099093760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767734.328094:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767734.328095:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767734.328097:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099093760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767734.328101:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099093888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767734.328113:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767734.328118:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767734/real 1775767734] req@ffff8b23864f0380 x1862022099093888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767734.328125:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099093888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767734.328130:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767734.328132:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767734.328135:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099093888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767734.328138:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099094016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767734.328149:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767734.328154:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767734/real 1775767734] req@ffff8b23864f2680 x1862022099094016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767734.328161:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099094016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767734.328167:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767734.328168:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767734.328170:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099094016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767735.223695:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767736.247712:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767737.271646:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767738.295652:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767739.319662:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767739.447541:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767739.447552:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767739.447557:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767739.447568:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5200 00000100:00080000:3.0:1775767739.447582:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767739.447589:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767739.447634:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099094144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767739.447662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767739.447665:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767739.447667:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767739.447670:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5200 00000100:00080000:3.0:1775767739.447672:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767739.447675:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767739.447684:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767739.447694:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099094272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767739.447703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767739.447705:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767739.447708:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767739.447711:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5200 00000100:00080000:3.0:1775767739.447713:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767739.447716:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767739.447719:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767739.447726:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099094400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767739.447734:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767739.447737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767739.447738:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767739.447743:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5200 00000100:00080000:3.0:1775767739.447746:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767739.447750:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767739.447755:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767739.447767:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099094528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767739.447777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767739.447781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767739.447783:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767739.447786:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5200 00000100:00080000:3.0:1775767739.447790:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767739.447794:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767739.447799:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767739.447811:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099094656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767739.448044:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099094144:192.168.202.128@tcp:250:kworker.0 00000800:00000100:0.0:1775767739.448124:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767739.448174:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767739/real 1775767739] req@ffff8b23864f2d80 x1862022099094144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767739.448195:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099094144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767739.448206:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767739.448211:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767739.448215:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099094144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767739.448224:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099094272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767739.448250:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767739.448266:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767739/real 1775767739] req@ffff8b23864f3100 x1862022099094272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767739.448280:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099094272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767739.448323:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767739.448328:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767739.448332:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099094272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767739.448338:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099094400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767739.448362:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767739.448375:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767739/real 1775767739] req@ffff8b23864f0700 x1862022099094400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767739.448387:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099094400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767739.448396:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767739.448400:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767739.448403:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099094400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767739.448411:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099094528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767739.448434:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767739.448445:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767739/real 1775767739] req@ffff8b23864f1180 x1862022099094528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767739.448514:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099094528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767739.448524:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767739.448528:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767739.448532:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099094528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767739.448538:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099094656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767739.448563:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767739.448574:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767739/real 1775767739] req@ffff8b23864f3b80 x1862022099094656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767739.448587:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099094656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767739.448595:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767739.448599:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767739.448602:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099094656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767740.343645:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767741.367603:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767742.391638:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767743.415620:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767744.439621:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767744.567516:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767744.567524:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767744.567528:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767744.567536:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5205 00000100:00080000:3.0:1775767744.567545:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767744.567551:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767744.567582:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099094784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767744.567605:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767744.567608:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767744.567610:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767744.567612:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5205 00000100:00080000:3.0:1775767744.567614:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767744.567617:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767744.567624:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767744.567633:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099094912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767744.567640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767744.567643:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767744.567644:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767744.567647:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5205 00000100:00080000:3.0:1775767744.567649:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767744.567652:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767744.567654:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767744.567660:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099095040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767744.567667:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767744.567669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767744.567671:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767744.567674:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5205 00000100:00080000:3.0:1775767744.567676:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767744.567679:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767744.567683:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767744.567687:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099094784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767744.567693:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099095168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767744.567700:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767744.567702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767744.567703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767744.567706:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5205 00000100:00080000:3.0:1775767744.567709:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767744.567710:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767744.567713:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767744.567722:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099095296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767744.567755:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767744.567794:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767744/real 1775767744] req@ffff8b23864f3480 x1862022099094784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767744.567808:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099094784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767744.567815:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767744.567818:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767744.567821:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099094784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767744.567827:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099094912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767744.567846:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767744.567858:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767744/real 1775767744] req@ffff8b23864f2a00 x1862022099094912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767744.567871:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099094912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767744.567879:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767744.567883:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767744.567887:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099094912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767744.567893:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099095040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767744.567924:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767744.567934:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767744/real 1775767744] req@ffff8b23864f0000 x1862022099095040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767744.567942:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099095040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767744.567947:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767744.567949:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767744.567952:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099095040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767744.567974:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099095168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767744.567988:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767744.567994:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767744/real 1775767744] req@ffff8b23864f3b80 x1862022099095168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767744.568003:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099095168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767744.568020:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767744.568023:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767744.568026:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099095168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767744.568033:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099095296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767744.568052:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767744.568062:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767744/real 1775767744] req@ffff8b23864f1180 x1862022099095296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767744.568074:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099095296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767744.568081:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767744.568085:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767744.568088:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099095296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767745.463642:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767746.487620:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767747.511719:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767748.535702:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767749.559613:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767749.687595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767749.687605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767749.687610:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767749.687621:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5211 00000100:00080000:3.0:1775767749.687636:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767749.687645:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767749.687712:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099095424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767749.687750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767749.687753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767749.687755:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767749.687759:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5211 00000100:00080000:3.0:1775767749.687762:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767749.687766:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767749.687776:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767749.687789:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099095552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767749.687799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767749.687802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767749.687804:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767749.687809:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5211 00000100:00080000:3.0:1775767749.687813:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767749.687815:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099095424:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767749.687817:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767749.687823:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767749.687833:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099095680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767749.687844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767749.687848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767749.687850:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767749.687855:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5211 00000100:00080000:3.0:1775767749.687860:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767749.687873:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767749.687880:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767749.687894:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099095808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767749.687895:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767749.687907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767749.687911:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767749.687913:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767749.687918:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5211 00000100:00080000:3.0:1775767749.687923:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767749.687927:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767749.687933:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767749.687946:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767749/real 1775767749] req@ffff8b23864f0700 x1862022099095424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767749.687948:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099095936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767749.687982:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099095424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767749.688004:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767749.688010:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767749.688015:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099095424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767749.688023:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099095552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767749.688044:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767749.688055:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767749/real 1775767749] req@ffff8b23864f3100 x1862022099095552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767749.688064:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099095552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767749.688070:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767749.688073:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767749.688075:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099095552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767749.688098:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099095680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767749.688110:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767749.688116:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767749/real 1775767749] req@ffff8b23864f2d80 x1862022099095680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767749.688124:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099095680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767749.688129:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767749.688131:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767749.688133:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099095680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767749.688137:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099095808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767749.688149:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767749.688155:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767749/real 1775767749] req@ffff8b23864f2680 x1862022099095808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767749.688162:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099095808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767749.688167:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767749.688169:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767749.688171:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099095808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767749.688175:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099095936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767749.688224:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767749.688235:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767749/real 1775767749] req@ffff8b23864f0380 x1862022099095936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767749.688244:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099095936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767749.688250:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767749.688252:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767749.688254:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099095936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767750.583685:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767751.607663:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767752.631735:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767753.655650:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767754.679691:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767754.807593:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767754.807603:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767754.807608:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767754.807619:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5216 00000100:00080000:0.0:1775767754.807632:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767754.807639:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767754.807683:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022099096064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767754.807711:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767754.807714:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767754.807715:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767754.807719:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5216 00000100:00080000:0.0:1775767754.807721:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767754.807723:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767754.807734:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767754.807745:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099096192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767754.807755:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767754.807758:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767754.807759:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767754.807763:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5216 00000100:00080000:0.0:1775767754.807765:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767754.807771:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767754.807771:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099096064:192.168.202.128@tcp:250:kworker.0 00000008:00100000:0.0:1775767754.807774:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767754.807785:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022099096320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767754.807796:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767754.807799:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767754.807800:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767754.807803:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5216 00000100:00080000:0.0:1775767754.807805:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767754.807807:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767754.807810:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767754.807818:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022099096448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767754.807824:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767754.807827:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767754.807829:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767754.807840:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5216 00000100:00080000:0.0:1775767754.807842:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767754.807848:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767754.807852:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767754.807861:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099096576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767754.807870:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767754.807937:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767754/real 1775767754] req@ffff8b2382dad880 x1862022099096064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767754.807960:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022099096064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767754.807973:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767754.807979:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767754.807985:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099096064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767754.808017:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099096192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767754.808044:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767754.808059:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767754/real 1775767754] req@ffff8b2382daca80 x1862022099096192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767754.808074:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099096192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767754.808083:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767754.808087:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767754.808091:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099096192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767754.808102:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099096320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767754.808125:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767754.808153:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767754/real 1775767754] req@ffff8b2382daf100 x1862022099096320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767754.808167:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022099096320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767754.808176:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767754.808180:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767754.808184:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099096320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767754.808191:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099096448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767754.808214:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767754.808226:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767754/real 1775767754] req@ffff8b2382dac000 x1862022099096448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767754.808238:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022099096448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767754.808247:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767754.808251:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767754.808255:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099096448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767754.808262:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099096576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767754.808285:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767754.808296:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767754/real 1775767754] req@ffff8b2382dac380 x1862022099096576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767754.808309:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099096576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767754.808318:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767754.808321:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767754.808325:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099096576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767755.703672:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767756.727618:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767757.751684:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767758.775639:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767759.799698:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767759.927608:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767759.927620:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767759.927625:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767759.927637:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5221 00000100:00080000:3.0:1775767759.927653:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767759.927660:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767759.927712:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099096704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767759.927746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767759.927750:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767759.927752:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767759.927756:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5221 00000100:00080000:3.0:1775767759.927760:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767759.927765:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767759.927776:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767759.927791:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099096832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767759.927800:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099096704:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767759.927807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767759.927811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767759.927813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767759.927817:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5221 00000100:00080000:3.0:1775767759.927820:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767759.927824:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767759.927830:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767759.927841:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099096960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767759.927851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767759.927854:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767759.927856:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767759.927860:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5221 00000100:00080000:3.0:1775767759.927864:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767759.927867:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767759.927873:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767759.927882:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767759.927886:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099097088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767759.927897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767759.927901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767759.927903:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767759.927908:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5221 00000100:00080000:3.0:1775767759.927911:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767759.927915:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767759.927921:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767759.927932:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767759/real 1775767759] req@ffff8b23864f0a80 x1862022099096704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767759.927936:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099097216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767759.927980:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099096704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767759.927990:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767759.927994:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767759.927998:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099096704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767759.928020:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099096832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767759.928037:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767759.928046:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767759/real 1775767759] req@ffff8b23864f1f80 x1862022099096832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767759.928054:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099096832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767759.928060:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767759.928062:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767759.928064:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099096832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767759.928068:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099096960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767759.928082:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767759.928106:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767759/real 1775767759] req@ffff8b23864f1c00 x1862022099096960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767759.928115:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099096960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767759.928126:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767759.928128:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767759.928130:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099096960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767759.928134:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099097088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767759.928148:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767759.928154:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767759/real 1775767759] req@ffff8b23864f0380 x1862022099097088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767759.928161:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099097088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767759.928166:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767759.928168:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767759.928170:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099097088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767759.928173:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099097216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767759.928184:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767759.928190:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767759/real 1775767759] req@ffff8b23864f2680 x1862022099097216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767759.928197:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099097216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767759.928202:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767759.928203:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767759.928206:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099097216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767760.823622:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767761.847769:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767762.871789:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767763.895641:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767764.919641:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767765.047563:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767765.047570:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767765.047575:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767765.047614:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5226 00000100:00080000:3.0:1775767765.047624:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767765.047641:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767765.047676:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099097344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767765.047701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767765.047704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767765.047706:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767765.047708:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5226 00000100:00080000:3.0:1775767765.047711:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767765.047713:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767765.047720:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767765.047732:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099097472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767765.047741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767765.047744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767765.047746:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767765.047747:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099097344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767765.047751:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5226 00000100:00080000:3.0:1775767765.047756:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767765.047759:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767765.047763:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767765.047772:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099097600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767765.047779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767765.047781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767765.047783:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767765.047785:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5226 00000100:00080000:3.0:1775767765.047787:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767765.047789:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767765.047792:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767765.047799:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099097728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767765.047805:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775767765.047808:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767765.047810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767765.047812:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767765.047817:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5226 00000100:00080000:3.0:1775767765.047820:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767765.047824:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767765.047827:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767765.047835:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099097856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767765.047850:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767765/real 1775767765] req@ffff8b23864f2d80 x1862022099097344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767765.047876:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099097344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767765.047884:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767765.047887:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767765.047890:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099097344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767765.047912:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099097472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767765.047936:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767765.047949:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767765/real 1775767765] req@ffff8b23864f3100 x1862022099097472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767765.047963:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099097472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767765.047971:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767765.047974:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767765.047976:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099097472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767765.047981:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099097600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767765.048000:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767765.048012:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767765/real 1775767765] req@ffff8b23864f0700 x1862022099097600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767765.048025:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099097600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767765.048047:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767765.048051:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767765.048055:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099097600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767765.048059:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099097728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767765.048076:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767765.048084:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767765/real 1775767765] req@ffff8b23864f1180 x1862022099097728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767765.048092:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099097728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767765.048097:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767765.048099:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767765.048101:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099097728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767765.048104:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099097856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767765.048115:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767765.048120:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767765/real 1775767765] req@ffff8b23864f3b80 x1862022099097856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767765.048127:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099097856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767765.048132:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767765.048134:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767765.048136:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099097856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767765.943835:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767766.967654:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767767.991683:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767769.015620:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767770.039634:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767770.167549:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767770.167565:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767770.167570:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767770.167581:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5231 00000100:00080000:3.0:1775767770.167594:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767770.167601:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767770.167643:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099097984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767770.167670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767770.167673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767770.167675:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767770.167680:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5231 00000100:00080000:3.0:1775767770.167683:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767770.167687:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767770.167698:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767770.167712:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099098112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767770.167725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767770.167729:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099097984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767770.167729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767770.167730:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767770.167735:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5231 00000100:00080000:3.0:1775767770.167738:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767770.167742:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767770.167747:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767770.167756:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099098240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767770.167770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767770.167774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767770.167775:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767770.167779:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5231 00000100:00080000:3.0:1775767770.167782:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767770.167786:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767770.167790:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767770.167799:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099098368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767770.167806:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767770.167808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767770.167810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767770.167812:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767770.167816:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5231 00000100:00080000:3.0:1775767770.167820:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767770.167823:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767770.167827:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767770.167837:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099098496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767770.167856:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767770/real 1775767770] req@ffff8b23864f0000 x1862022099097984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767770.167877:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099097984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767770.167889:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767770.167893:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767770.167896:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099097984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767770.169030:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099098112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767770.169055:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767770.169066:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767770/real 1775767770] req@ffff8b23864f2a00 x1862022099098112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767770.169075:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099098112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767770.169081:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767770.169084:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767770.169086:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099098112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767770.169091:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099098240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767770.169103:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767770.169109:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767770/real 1775767770] req@ffff8b23864f3480 x1862022099098240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767770.169117:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099098240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767770.169122:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767770.169124:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767770.169125:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099098240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767770.169129:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099098368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767770.170133:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767770.170142:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767770/real 1775767770] req@ffff8b23864f3b80 x1862022099098368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767770.170150:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099098368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767770.170156:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767770.170158:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767770.170161:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099098368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767770.170165:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099098496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767770.170177:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767770.170183:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767770/real 1775767770] req@ffff8b23864f1180 x1862022099098496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767770.170191:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099098496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767770.170196:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767770.170198:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767770.170200:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099098496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767771.063637:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767772.087673:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767773.111625:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767774.135601:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767775.159700:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767775.287531:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767775.287546:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767775.287551:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767775.287565:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5236 00000100:00080000:3.0:1775767775.287589:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767775.287598:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767775.287652:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099098624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767775.287685:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767775.287689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767775.287690:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767775.287694:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5236 00000100:00080000:3.0:1775767775.287698:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767775.287701:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767775.287711:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767775.287724:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099098752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767775.287737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767775.287741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767775.287743:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767775.287749:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5236 00000100:00080000:3.0:1775767775.287754:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767775.287759:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767775.287766:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767775.287774:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099098624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767775.287781:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099098880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767775.287789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767775.287792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767775.287793:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767775.287797:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5236 00000100:00080000:3.0:1775767775.287800:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767775.287802:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767775.287807:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767775.287820:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099099008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767775.287839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767775.287843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767775.287845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775767775.287849:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767775.287853:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5236 00000100:00080000:3.0:1775767775.287859:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767775.287865:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767775.287870:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767775.287882:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099099136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767775.287899:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767775/real 1775767775] req@ffff8b23864f0700 x1862022099098624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767775.287916:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099098624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767775.287954:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767775.287958:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767775.287962:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099098624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767775.287968:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099098752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767775.287985:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767775.287994:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767775/real 1775767775] req@ffff8b23864f3100 x1862022099098752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767775.288002:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099098752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767775.288012:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767775.288015:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767775.288017:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099098752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767775.288039:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099098880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767775.288050:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767775.288055:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767775/real 1775767775] req@ffff8b23864f2d80 x1862022099098880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767775.288062:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099098880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767775.288068:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767775.288069:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767775.288071:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099098880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767775.288075:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099099008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767775.288087:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767775.288092:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767775/real 1775767775] req@ffff8b23864f2680 x1862022099099008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767775.288099:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099099008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767775.288104:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767775.288106:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767775.288108:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099099008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767775.288112:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099099136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767775.288123:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767775.288128:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767775/real 1775767775] req@ffff8b23864f0380 x1862022099099136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767775.288135:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099099136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767775.288140:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767775.288142:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767775.288143:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099099136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767776.183688:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767777.207641:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767778.231667:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767779.255693:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767780.279640:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767780.407612:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767780.407623:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767780.407628:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767780.407638:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5241 00000100:00080000:1.0:1775767780.407652:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767780.407659:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767780.407707:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099099264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767780.407735:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767780.407738:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767780.407739:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767780.407743:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5241 00000100:00080000:1.0:1775767780.407745:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767780.407747:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767780.407767:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767780.407778:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099099392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767780.407789:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767780.407791:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099099264:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767780.407792:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767780.407795:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767780.407803:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5241 00000100:00080000:1.0:1775767780.407805:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767780.407808:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767780.407811:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767780.407822:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099099520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767780.407829:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767780.407831:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767780.407833:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767780.407836:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5241 00000100:00080000:1.0:1775767780.407838:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767780.407840:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767780.407843:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767780.407849:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099099648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767780.407855:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767780.407858:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767780.407859:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767780.407863:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5241 00000100:00080000:1.0:1775767780.407866:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767780.407869:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767780.407872:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767780.407882:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099099776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767780.407938:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099099392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767780.407974:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099099520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767780.407991:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099099648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767780.408006:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099099776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767781.303727:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767782.327716:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767783.351616:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775767783.479907:0:2363:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:3.0:1775767783.479955:0:2363:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775767783.479972:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775767783.480016:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775767783.480025:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775767783.480036:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:3.0:1775767783.480046:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775767783.480076:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767780/real 1775767783] req@ffff8b23864fdf80 x1862022099099392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767783.480105:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099099392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767783.480123:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767783.480129:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767783.480133:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099099392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767783.480142:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767780/real 1775767783] req@ffff8b23864ff800 x1862022099099520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767783.480155:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099099520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767783.480163:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767783.480167:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767783.480170:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099099520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767783.480178:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767780/real 1775767783] req@ffff8b23864fed80 x1862022099099648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767783.480190:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099099648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767783.480198:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767783.480202:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767783.480206:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099099648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767783.480213:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767780/real 1775767783] req@ffff8b23864fc000 x1862022099099776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767783.480226:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099099776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767783.480252:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767783.480255:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767783.480257:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099099776:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767783.480262:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767780/real 1775767783] req@ffff8b23864fc380 x1862022099099264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767783.480281:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099099264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767783.480290:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767783.480295:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767783.480298:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099099264:192.168.202.128@tcp:250:kworker.0 00010000:00010000:1.0:1775767784.375664:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767785.399692:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767785.527594:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767785.527605:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767785.527609:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767785.527620:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5246 00000100:00080000:0.0:1775767785.527633:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767785.527765:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767785.527820:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099099904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767785.527880:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767785.527886:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767785.527889:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767785.527895:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5246 00000100:00080000:0.0:1775767785.527907:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767785.527914:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767785.527925:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767785.527940:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099100032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767785.527950:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099099904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767785.527959:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767785.527964:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767785.527966:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767785.527972:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5246 00000100:00080000:0.0:1775767785.527976:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767785.527981:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767785.527987:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767785.528004:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099100160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767785.528014:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767785.528018:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767785.528020:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767785.528025:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5246 00000100:00080000:0.0:1775767785.528029:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775767785.528035:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775767785.528036:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767785.528044:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767785.528058:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099100288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767785.528075:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767785.528078:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767785.528080:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767785.528084:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5246 00000100:00080000:0.0:1775767785.528088:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767785.528090:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767785/real 1775767785] req@ffff8b2382dae680 x1862022099099904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767785.528095:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767785.528103:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767785.528120:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099100416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767785.528123:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099099904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767785.528154:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767785.528160:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767785.528166:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099099904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767785.528226:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099100032:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767785.528254:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767785.528267:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767785/real 1775767785] req@ffff8b2382dad500 x1862022099100032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767785.528279:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099100032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767785.528286:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767785.528289:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767785.528307:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099100032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767785.528314:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099100160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767785.528334:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767785.528352:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767785/real 1775767785] req@ffff8b2382daea00 x1862022099100160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767785.528364:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099100160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767785.528496:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767785.528503:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767785.528508:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099100160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767785.528516:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099100288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767785.528542:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767785.528575:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767785/real 1775767785] req@ffff8b2382daed80 x1862022099100288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767785.528589:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099100288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767785.528598:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767785.528601:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767785.528605:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099100288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767785.528611:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099100416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767785.528635:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767785.528646:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767785/real 1775767785] req@ffff8b2382daf480 x1862022099100416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767785.528657:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099100416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767785.528665:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767785.528668:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767785.528672:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099100416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767786.423634:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767787.447724:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767788.471661:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767789.495663:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767790.519692:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767790.647620:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767790.647629:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767790.647633:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767790.647642:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5252 00000100:00080000:3.0:1775767790.647669:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767790.647675:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767790.647725:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099100544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767790.647752:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767790.647757:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767790.647759:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767790.647764:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5252 00000100:00080000:3.0:1775767790.647769:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767790.647773:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767790.647784:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767790.647828:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099100544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767790.647830:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099100672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767790.647851:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767790.647854:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767790.647855:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767790.647859:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5252 00000100:00080000:3.0:1775767790.647863:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767790.647867:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767790.647872:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767790.647884:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099100800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767790.647891:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767790.647893:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767790.647895:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767790.647900:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5252 00000100:00080000:3.0:1775767790.647903:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767790.647905:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767790.647908:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767790.647918:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099100928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767790.647924:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767790.647926:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767790.647927:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775767790.647934:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767790.647939:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5252 00000100:00080000:3.0:1775767790.647942:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767790.647944:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767790.647947:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767790.647963:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099101056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767790.647992:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767790/real 1775767790] req@ffff8b23864f1c00 x1862022099100544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767790.648010:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099100544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767790.648020:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767790.648024:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767790.648028:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099100544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767790.648052:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099100672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767790.648068:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767790.648178:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767790/real 1775767790] req@ffff8b23864f1f80 x1862022099100672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767790.648201:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099100672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767790.648209:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767790.648212:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767790.648215:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099100672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767790.648222:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099100800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767790.648265:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767790.648299:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767790/real 1775767790] req@ffff8b23864f0a80 x1862022099100800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767790.648307:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099100800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767790.648316:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767790.648319:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767790.648321:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099100800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767790.648326:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099100928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767790.648342:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767790.648349:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767790/real 1775767790] req@ffff8b23864f0380 x1862022099100928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767790.648357:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099100928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767790.648372:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767790.648375:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767790.648379:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099100928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767790.648385:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099101056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767790.648407:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767790.648418:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767790/real 1775767790] req@ffff8b23864f2680 x1862022099101056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767790.648431:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099101056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767790.648440:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767790.648444:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767790.648448:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099101056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767791.543673:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767792.567621:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767793.591616:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767794.615625:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767795.639632:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767795.767582:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767795.767595:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767795.767601:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767795.767621:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5257 00000100:00080000:0.0:1775767795.767636:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767795.767644:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767795.767694:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022099101184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767795.767727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767795.767732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767795.767734:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767795.767756:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5257 00000100:00080000:0.0:1775767795.767760:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767795.767765:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767795.767777:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767795.767797:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099101184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767795.767806:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022099101312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767795.767822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767795.767826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767795.767828:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767795.767834:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5257 00000100:00080000:0.0:1775767795.767840:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767795.767845:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767795.767850:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767795.767865:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022099101440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767795.767872:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775767795.767883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767795.767888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767795.767890:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767795.767896:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5257 00000100:00080000:0.0:1775767795.768014:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767795.768020:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767795.768027:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767795.768056:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022099101568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767795.768069:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775767795.768076:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767795/real 1775767795] req@ffff8b2382dafb80 x1862022099101184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767795.768077:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767795.768082:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767795.768099:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5257 00000100:00080000:0.0:1775767795.768113:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767795.768121:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767795.768128:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767795.768133:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022099101184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767795.768144:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1775767795.768150:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099101696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767795.768150:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767795.768157:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099101184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767795.768206:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099101312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767795.768267:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767795.768284:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767795/real 1775767795] req@ffff8b2382dadf80 x1862022099101312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767795.768294:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022099101312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767795.768299:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767795.768302:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767795.768304:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099101312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767795.768309:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099101440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767795.768326:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767795.768332:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767795/real 1775767795] req@ffff8b2382dace00 x1862022099101440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767795.768340:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022099101440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767795.768345:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767795.768347:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767795.768349:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099101440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767795.768354:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099101568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767795.768391:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767795.768398:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767795/real 1775767795] req@ffff8b2382dac700 x1862022099101568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767795.768405:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022099101568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767795.768410:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767795.768412:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767795.768414:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099101568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767795.768419:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099101696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767795.768430:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767795.768435:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767795/real 1775767795] req@ffff8b2382daf480 x1862022099101696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767795.768442:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099101696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767795.768447:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767795.768449:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767795.768490:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099101696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767796.663696:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767797.687599:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767798.711653:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767799.735609:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767800.759728:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767800.887538:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767800.887549:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767800.887554:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767800.887565:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5262 00000100:00080000:0.0:1775767800.887577:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767800.887585:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767800.887638:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099101824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767800.887668:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767800.887672:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767800.887673:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767800.887676:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5262 00000100:00080000:0.0:1775767800.887681:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767800.887684:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767800.887714:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767800.887729:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099101952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767800.887739:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767800.887742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767800.887744:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767800.887747:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5262 00000100:00080000:0.0:1775767800.887750:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767800.887755:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099101824:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767800.887756:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767800.887761:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767800.887773:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099102080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767800.887780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767800.887783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767800.887784:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767800.887787:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5262 00000100:00080000:0.0:1775767800.887789:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767800.887791:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767800.887795:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767800.887803:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099102208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767800.887814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767800.887817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767800.887819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767800.887821:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5262 00000100:00080000:0.0:1775767800.887823:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767800.887825:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767800.887828:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767800.887838:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099102336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767800.887861:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767800.887925:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767800/real 1775767800] req@ffff8b2382daed80 x1862022099101824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767800.887948:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099101824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767800.887961:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767800.887965:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767800.887970:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099101824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767800.887979:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099101952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767800.888006:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767800.888020:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767800/real 1775767800] req@ffff8b2382daea00 x1862022099101952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767800.888033:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099101952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767800.888041:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767800.888045:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767800.888048:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099101952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767800.888079:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099102080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767800.888101:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767800.888111:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767800/real 1775767800] req@ffff8b2382dad500 x1862022099102080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767800.888131:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099102080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767800.888139:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767800.888143:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767800.888147:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099102080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767800.888153:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099102208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767800.888172:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767800.888181:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767800/real 1775767800] req@ffff8b2382dae680 x1862022099102208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767800.888193:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099102208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767800.888201:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767800.888205:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767800.888208:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099102208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767800.888215:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099102336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767800.888234:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767800.888244:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767800/real 1775767800] req@ffff8b2382dac380 x1862022099102336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767800.888255:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099102336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767800.888271:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767800.888274:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767800.888279:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099102336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767801.783701:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767802.807710:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767803.831617:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767804.855723:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767805.879613:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767806.007586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767806.007598:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767806.007603:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767806.007614:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5267 00000100:00080000:0.0:1775767806.007635:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767806.007673:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767806.007712:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022099102464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767806.007743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767806.007747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767806.007749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767806.007753:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5267 00000100:00080000:0.0:1775767806.007757:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767806.007762:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767806.007772:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767806.007791:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022099102592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767806.007804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767806.007808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767806.007811:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767806.007816:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5267 00000100:00080000:0.0:1775767806.007821:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767806.007830:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767806.007830:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099102464:192.168.202.128@tcp:250:kworker.0 00000008:00100000:0.0:1775767806.007855:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767806.007875:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099102720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767806.007889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767806.007894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767806.007896:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767806.007900:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5267 00000100:00080000:0.0:1775767806.007905:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767806.007910:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767806.007916:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767806.007934:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022099102848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767806.007935:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775767806.007950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767806.007953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767806.007957:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767806.007962:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5267 00000100:00080000:0.0:1775767806.007967:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767806.007971:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767806.007977:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767806.007999:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099102976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767806.007999:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767806/real 1775767806] req@ffff8b2382dac000 x1862022099102464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767806.008073:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022099102464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767806.008086:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767806.008090:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767806.008094:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099102464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767806.008101:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099102592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767806.008125:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767806.008134:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767806/real 1775767806] req@ffff8b2382daf100 x1862022099102592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767806.008143:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022099102592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767806.008152:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767806.008155:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767806.008157:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099102592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767806.008183:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099102720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767806.008195:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767806.008200:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767806/real 1775767806] req@ffff8b2382daca80 x1862022099102720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767806.008207:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099102720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767806.008213:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767806.008215:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767806.008217:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099102720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767806.008221:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099102848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767806.008232:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767806.008237:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767806/real 1775767806] req@ffff8b2382dad880 x1862022099102848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767806.008244:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022099102848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767806.008248:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767806.008250:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767806.008253:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099102848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767806.008258:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099102976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767806.008268:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767806.008273:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767806/real 1775767806] req@ffff8b2382dac380 x1862022099102976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767806.008280:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099102976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767806.008286:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767806.008288:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767806.008290:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099102976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767806.903691:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767807.927655:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767808.951698:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767809.975625:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767810.999730:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767811.127559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767811.127569:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767811.127574:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767811.127585:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5272 00000100:00080000:0.0:1775767811.127620:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767811.127628:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767811.127677:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099103104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767811.127701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767811.127704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767811.127706:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767811.127709:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5272 00000100:00080000:0.0:1775767811.127712:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767811.127714:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767811.127724:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767811.127733:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099103232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767811.127747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767811.127750:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767811.127752:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767811.127754:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099103104:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767811.127758:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5272 00000100:00080000:0.0:1775767811.127763:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767811.127766:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767811.127769:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767811.127778:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099103360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767811.127785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767811.127788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767811.127789:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767811.127793:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5272 00000100:00080000:0.0:1775767811.127795:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767811.127797:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767811.127800:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767811.127807:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099103488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767811.127814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767811.127817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767811.127818:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767811.127821:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5272 00000100:00080000:0.0:1775767811.127824:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767811.127827:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767811.127831:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767811.127831:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767811.127842:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099103616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767811.127877:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767811/real 1775767811] req@ffff8b2382dae680 x1862022099103104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767811.127893:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099103104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767811.127901:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767811.127905:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767811.127908:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099103104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767811.127937:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099103232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767811.127963:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767811.127977:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767811/real 1775767811] req@ffff8b2382dad500 x1862022099103232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767811.127987:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099103232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767811.127993:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767811.127995:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767811.127998:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099103232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767811.128002:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099103360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767811.128017:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767811.128024:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767811/real 1775767811] req@ffff8b2382daea00 x1862022099103360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767811.128031:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099103360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767811.128036:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767811.128038:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767811.128041:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099103360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767811.128047:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099103488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767811.128068:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767811.128079:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767811/real 1775767811] req@ffff8b2382daed80 x1862022099103488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767811.128100:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099103488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767811.128106:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767811.128108:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767811.128111:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099103488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767811.128115:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099103616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767811.128130:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767811.128136:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767811/real 1775767811] req@ffff8b2382daf480 x1862022099103616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767811.128143:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099103616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767811.128149:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767811.128150:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767811.128152:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099103616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767812.023621:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767813.047693:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767814.071771:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767815.095626:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767816.119681:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767816.247616:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767816.247627:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767816.247633:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767816.247643:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5277 00000100:00080000:1.0:1775767816.247656:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767816.247664:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767816.247708:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099103744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767816.247733:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767816.247736:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767816.247738:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767816.247740:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5277 00000100:00080000:1.0:1775767816.247743:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767816.247745:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767816.247754:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767816.247767:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099103872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767816.247780:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775767816.247784:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099103744:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767816.247786:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767816.247790:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767816.247798:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5277 00000100:00080000:1.0:1775767816.247802:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767816.247808:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767816.247814:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767816.247830:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099104000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767816.247841:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767816.247845:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767816.247849:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767816.247853:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5277 00000100:00080000:1.0:1775767816.247857:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767816.247863:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775767816.247866:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:1.0:1775767816.247872:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767816.247889:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099104128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767816.247900:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767816.247904:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767816.247906:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767816.247910:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5277 00000100:00080000:1.0:1775767816.247915:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767816.247919:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767816.247925:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767816.247942:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099104256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767816.247949:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767816/real 1775767816] req@ffff8b23864fc380 x1862022099103744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767816.247985:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099103744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767816.248006:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767816.248011:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767816.248016:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099103744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767816.248049:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099103872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767816.248077:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767816.248091:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767816/real 1775767816] req@ffff8b23864fc000 x1862022099103872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767816.248110:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099103872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767816.248117:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767816.248119:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767816.248121:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099103872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767816.248126:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099104000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767816.248143:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767816.248150:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767816/real 1775767816] req@ffff8b23864fed80 x1862022099104000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767816.248158:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099104000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767816.248169:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767816.248171:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767816.248174:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099104000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767816.248177:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099104128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767816.248189:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767816.248194:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767816/real 1775767816] req@ffff8b23864ff800 x1862022099104128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767816.248202:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099104128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767816.248207:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767816.248209:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767816.248211:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099104128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767816.248214:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099104256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767816.248234:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767816.248240:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767816/real 1775767816] req@ffff8b23864fdf80 x1862022099104256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767816.248247:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099104256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767816.248253:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767816.248255:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767816.248258:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099104256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767817.143640:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767818.167700:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767819.191613:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767820.215629:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767821.239683:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767821.367578:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767821.367588:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767821.367593:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767821.367603:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5282 00000100:00080000:1.0:1775767821.367615:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767821.367621:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767821.367683:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099104384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767821.367713:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767821.367716:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767821.367717:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767821.367720:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5282 00000100:00080000:1.0:1775767821.367723:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767821.367725:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767821.367735:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767821.367744:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099104512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767821.367753:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767821.367756:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767821.367757:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767821.367761:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5282 00000100:00080000:1.0:1775767821.367765:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767821.367766:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099104384:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767821.367772:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767821.367778:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767821.367793:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099104640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767821.367804:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767821.367808:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767821.367810:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767821.367816:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5282 00000100:00080000:1.0:1775767821.367820:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767821.367824:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767821.367830:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767821.367840:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775767821.367844:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099104768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767821.367855:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767821.367859:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767821.367861:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767821.367865:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5282 00000100:00080000:1.0:1775767821.367869:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767821.367873:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767821.367878:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767821.367889:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767821/real 1775767821] req@ffff8b23864fce00 x1862022099104384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767821.367897:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099104896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767821.367910:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099104384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767821.367920:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767821.367924:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767821.367928:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099104384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767821.367935:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099104512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767821.367953:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767821.367961:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767821/real 1775767821] req@ffff8b23864fd180 x1862022099104512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767821.367970:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099104512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767821.367975:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767821.367978:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767821.367980:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099104512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767821.368003:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099104640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767821.368015:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767821.368021:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767821/real 1775767821] req@ffff8b23864fea00 x1862022099104640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767821.368029:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099104640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767821.368034:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767821.368036:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767821.368038:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099104640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767821.368041:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099104768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767821.368053:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767821.368058:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767821/real 1775767821] req@ffff8b23864fca80 x1862022099104768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767821.368065:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099104768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767821.368070:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767821.368072:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767821.368074:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099104768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767821.368077:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099104896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767821.368087:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767821.368092:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767821/real 1775767821] req@ffff8b23864fe680 x1862022099104896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767821.368113:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099104896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767821.368118:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767821.368120:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767821.368122:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099104896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767822.263645:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767823.287619:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767824.311643:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767825.335673:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767826.359664:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767826.487566:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767826.487579:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767826.487585:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767826.487597:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5287 00000100:00080000:1.0:1775767826.487612:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767826.487621:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767826.487666:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099105024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767826.487696:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767826.487699:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767826.487701:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767826.487704:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5287 00000100:00080000:1.0:1775767826.487707:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767826.487710:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767826.487720:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767826.487729:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099105152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767826.487739:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767826.487742:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767826.487744:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767826.487747:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5287 00000100:00080000:1.0:1775767826.487749:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767826.487752:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767826.487755:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767826.487764:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099105280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767826.487764:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099105024:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767826.487778:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767826.487780:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767826.487781:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767826.487784:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5287 00000100:00080000:1.0:1775767826.487787:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767826.487789:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767826.487791:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767826.487801:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099105408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767826.487807:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767826.487809:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767826.487810:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767826.487812:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5287 00000100:00080000:1.0:1775767826.487815:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767826.487817:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767826.487819:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767826.487829:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099105536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767826.487847:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767826.487894:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767826/real 1775767826] req@ffff8b23864fdc00 x1862022099105024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767826.487918:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099105024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767826.487930:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767826.487935:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767826.487940:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099105024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767826.487948:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099105152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767826.487970:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767826.487980:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767826/real 1775767826] req@ffff8b23864fd500 x1862022099105152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767826.487994:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099105152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767826.488000:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767826.488003:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767826.488005:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099105152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767826.488031:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099105280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767826.488042:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767826.488049:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767826/real 1775767826] req@ffff8b23864ff100 x1862022099105280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767826.488057:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099105280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767826.488062:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767826.488063:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767826.488065:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099105280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767826.488068:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099105408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767826.488080:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767826.488085:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767826/real 1775767826] req@ffff8b23864fe300 x1862022099105408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767826.488093:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099105408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767826.488103:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767826.488105:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767826.488108:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099105408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767826.488111:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099105536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767826.488120:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767826.488125:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767826/real 1775767826] req@ffff8b23864ffb80 x1862022099105536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767826.488132:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099105536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767826.488137:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767826.488139:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767826.488141:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099105536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767827.383609:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767828.407602:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767829.431651:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767830.455673:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767831.479595:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767831.607593:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767831.607605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767831.607610:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767831.607620:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5293 00000100:00080000:0.0:1775767831.607633:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767831.607639:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767831.607707:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022099105664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767831.607736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767831.607740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767831.607741:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767831.607744:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5293 00000100:00080000:0.0:1775767831.607747:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767831.607750:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767831.607761:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767831.607773:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022099105792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767831.607783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767831.607788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767831.607791:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767831.607792:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099105664:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767831.607796:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5293 00000100:00080000:0.0:1775767831.607801:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767831.607804:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767831.607807:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767831.607815:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022099105920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767831.607821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767831.607824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767831.607825:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767831.607827:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5293 00000100:00080000:0.0:1775767831.607830:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767831.607832:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767831.607835:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767831.607843:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022099106048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767831.607852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767831.607856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767831.607858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767831.607862:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5293 00000100:00080000:0.0:1775767831.607866:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767831.607871:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767831.607878:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767831.607878:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767831.607920:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767831/real 1775767831] req@ffff8b2382dac700 x1862022099105664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767831.607937:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022099105664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767831.607946:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767831.607950:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767831.607953:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099105664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767831.607978:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099105792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767831.607994:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767831.608003:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767831/real 1775767831] req@ffff8b2382dace00 x1862022099105792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767831.608011:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022099105792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767831.608017:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767831.608020:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767831.608022:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099105792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767831.608028:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099105920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767831.608042:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767831.608050:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767831/real 1775767831] req@ffff8b2382dadf80 x1862022099105920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767831.608053:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099106176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767831.608062:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022099105920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767831.608117:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767831.608119:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767831.608126:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099105920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767831.608134:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099106048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767831.608149:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767831.608155:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767831/real 1775767831] req@ffff8b2382dafb80 x1862022099106048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767831.608163:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022099106048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767831.608169:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767831.608172:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767831.608175:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099106048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767831.608189:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099106176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767831.608199:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767831.608205:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767831/real 1775767831] req@ffff8b2382daf480 x1862022099106176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767831.608212:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099106176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767831.608218:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767831.608220:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767831.608222:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099106176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767832.503598:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767833.527602:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767834.551617:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767835.575686:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767836.599608:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767836.727638:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767836.727648:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767836.727654:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767836.727669:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5298 00000100:00080000:3.0:1775767836.727685:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767836.727693:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767836.727748:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099106304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767836.727780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767836.727783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767836.727785:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767836.727788:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5298 00000100:00080000:3.0:1775767836.727791:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767836.727794:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767836.727804:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767836.727817:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099106432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767836.727830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767836.727834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767836.727836:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767836.727841:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5298 00000100:00080000:3.0:1775767836.727846:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767836.727853:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099106304:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767836.727854:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767836.727867:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767836.727882:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099106560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767836.727897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767836.727900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767836.727902:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767836.727905:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5298 00000100:00080000:3.0:1775767836.727909:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767836.727912:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767836.727915:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767836.727922:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099106688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767836.727929:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767836.727931:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767836.727933:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767836.727935:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5298 00000100:00080000:3.0:1775767836.727939:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767836.727945:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775767836.727953:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:3.0:1775767836.727956:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767836.727964:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099106816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767836.728005:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767836/real 1775767836] req@ffff8b23864f2d80 x1862022099106304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767836.728029:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099106304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767836.728040:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767836.728045:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767836.728048:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099106304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767836.728058:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099106432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767836.728085:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767836.728099:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767836/real 1775767836] req@ffff8b23864f3100 x1862022099106432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767836.728112:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099106432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767836.728120:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767836.728123:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767836.728127:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099106432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767836.728152:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099106560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767836.728175:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767836.728186:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767836/real 1775767836] req@ffff8b23864f0700 x1862022099106560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767836.728199:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099106560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767836.728207:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767836.728223:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767836.728227:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099106560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767836.728234:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099106688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767836.728255:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767836.728279:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767836/real 1775767836] req@ffff8b23864f1180 x1862022099106688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767836.728292:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099106688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767836.728299:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767836.728303:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767836.728307:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099106688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767836.728313:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099106816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767836.728334:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767836.728381:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767836/real 1775767836] req@ffff8b23864f3b80 x1862022099106816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767836.728395:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099106816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767836.728400:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767836.728403:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767836.728405:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099106816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767837.623708:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767838.647600:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767839.671612:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767840.695689:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767841.719658:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767841.847610:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767841.847618:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767841.847622:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767841.847631:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5303 00000100:00080000:0.0:1775767841.847641:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767841.847646:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767841.847693:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099106944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767841.847740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767841.847744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767841.847746:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767841.847750:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5303 00000100:00080000:0.0:1775767841.847755:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767841.847761:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767841.847786:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767841.847814:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099107072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767841.847835:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099106944:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767841.847859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767841.847865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767841.847867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767841.847877:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5303 00000100:00080000:0.0:1775767841.847883:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767841.847889:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767841.847896:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767841.847916:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099107200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767841.847928:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767841.847933:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767841.847936:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775767841.847944:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775767841.847945:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5303 00000100:00080000:0.0:1775767841.847954:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767841.847960:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767841.847968:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767841.847989:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099107328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767841.848005:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775767841.848005:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767841/real 1775767841] req@ffff8b2382daed80 x1862022099106944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767841.848026:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767841.848035:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767841.848059:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5303 00000100:00100000:3.0:1775767841.848068:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099106944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767841.848069:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767841.848091:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:3.0:1775767841.848101:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1775767841.848104:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775767841.848109:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767841.848119:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099106944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767841.848152:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099107456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767841.848160:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099107072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767841.848234:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767841.848268:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767841/real 1775767841] req@ffff8b2382daea00 x1862022099107072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767841.848279:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099107072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767841.848286:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767841.848289:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767841.848348:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099107072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767841.848355:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099107200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767841.848378:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767841.848388:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767841/real 1775767841] req@ffff8b2382dad500 x1862022099107200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767841.848398:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099107200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767841.848411:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767841.848414:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767841.848417:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099107200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767841.848422:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099107328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767841.848437:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767841.848444:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767841/real 1775767841] req@ffff8b2382dae680 x1862022099107328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767841.848453:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099107328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767841.848459:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767841.848461:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767841.848463:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099107328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767841.848504:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099107456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767841.848522:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767841.848530:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767841/real 1775767841] req@ffff8b2382dac380 x1862022099107456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767841.848559:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099107456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767841.848565:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767841.848568:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767841.848571:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099107456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767842.743686:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767843.767618:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767844.791610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767845.815621:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767846.839694:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767846.967583:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767846.967594:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767846.967599:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767846.967610:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5308 00000100:00080000:0.0:1775767846.967623:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767846.967631:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767846.967676:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022099107584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767846.967703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767846.967706:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767846.967707:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767846.967710:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5308 00000100:00080000:0.0:1775767846.967713:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767846.967715:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767846.967724:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767846.967732:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099107712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767846.967742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767846.967748:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767846.967749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767846.967753:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5308 00000100:00080000:0.0:1775767846.967757:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767846.967761:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099107584:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767846.967766:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767846.967774:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767846.967790:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022099107840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767846.967803:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767846.967807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767846.967809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767846.967814:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5308 00000100:00080000:0.0:1775767846.967819:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767846.967824:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767846.967831:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767846.967842:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022099107968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767846.967850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767846.967853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767846.967854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767846.967857:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5308 00000100:00080000:0.0:1775767846.967860:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767846.967863:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767846.967868:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767846.967878:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099107712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767846.967883:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099108096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767846.967921:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099107840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767846.967954:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099107968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767846.967983:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099108096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767847.863610:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767848.887685:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767849.911617:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767850.039685:0:2364:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775767850.039723:0:2364:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767850.039736:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767850.039768:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767850.039774:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767850.039781:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767850.039786:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:3.0:1775767850.039892:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767846/real 1775767850] req@ffff8b2382daf100 x1862022099107840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767850.039949:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022099107840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767850.039966:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767850.039972:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767850.039978:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099107840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767850.039989:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767846/real 1775767850] req@ffff8b2382dac000 x1862022099107968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767850.040005:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022099107968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767850.040021:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767850.040025:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767850.040029:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099107968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767850.040037:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767846/real 1775767850] req@ffff8b2382dac380 x1862022099108096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767850.040050:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099108096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767850.040059:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767850.040063:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767850.040067:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099108096:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767850.040076:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767846/real 1775767850] req@ffff8b2382dad880 x1862022099107584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767850.040089:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022099107584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767850.040108:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767850.040113:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767850.040118:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099107584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767850.040128:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767846/real 1775767850] req@ffff8b2382daca80 x1862022099107712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767850.040142:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099107712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767850.040152:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767850.040156:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767850.040159:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099107712:192.168.202.128@tcp:38:kworker.0 00010000:00010000:3.0:1775767850.935629:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767851.959599:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767852.087557:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767852.087566:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767852.087569:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767852.087578:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5313 00000100:00080000:3.0:1775767852.087599:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767852.087604:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767852.087644:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099108224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767852.087669:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767852.087672:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767852.087673:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767852.087677:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5313 00000100:00080000:3.0:1775767852.087681:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767852.087697:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767852.087708:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767852.087723:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099108224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767852.087725:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099108352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767852.087743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767852.087747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767852.087749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767852.087753:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5313 00000100:00080000:3.0:1775767852.087757:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767852.087762:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767852.087767:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767852.087792:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099108480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767852.087802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767852.087806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767852.087810:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775767852.087812:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775767852.087818:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5313 00000100:00080000:3.0:1775767852.087823:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767852.087828:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767852.087833:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767852.087857:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767852/real 1775767852] req@ffff8b23864f3480 x1862022099108224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767852.087865:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099108608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767852.087883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767852.087893:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767852.087896:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767852.087902:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5313 00000100:00080000:3.0:1775767852.087907:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767852.087909:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099108224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767852.087915:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767852.087925:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775767852.087941:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767852.087948:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767852.087953:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099108736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767852.087956:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099108224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767852.087991:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099108352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767852.088030:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767852.088043:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767852/real 1775767852] req@ffff8b23864f2a00 x1862022099108352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767852.088065:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099108352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767852.088073:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767852.088077:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767852.088081:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099108352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767852.088088:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099108480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767852.088113:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767852.088125:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767852/real 1775767852] req@ffff8b23864f0000 x1862022099108480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767852.088138:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099108480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767852.088146:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767852.088149:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767852.088152:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099108480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767852.088158:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099108608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767852.088183:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767852.088207:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767852/real 1775767852] req@ffff8b23864f3b80 x1862022099108608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767852.088220:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099108608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767852.088227:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767852.088230:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767852.088233:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099108608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767852.089080:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099108736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767852.089118:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767852.089136:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767852/real 1775767852] req@ffff8b23864f1180 x1862022099108736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767852.089176:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099108736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767852.089186:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767852.089190:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767852.089205:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099108736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767852.983586:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767854.007608:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767855.031716:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767856.055660:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767857.079643:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767857.207536:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767857.207547:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767857.207552:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767857.207563:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5318 00000100:00080000:3.0:1775767857.207575:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767857.207581:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767857.207629:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099108864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767857.207654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767857.207657:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767857.207659:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767857.207661:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5318 00000100:00080000:3.0:1775767857.207664:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767857.207667:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767857.207674:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767857.207684:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099108992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767857.207693:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767857.207696:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767857.207698:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767857.207701:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5318 00000100:00080000:3.0:1775767857.207706:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767857.207708:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767857.207710:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767857.207717:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099109120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767857.207725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767857.207729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767857.207730:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767857.207733:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5318 00000100:00080000:3.0:1775767857.207736:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767857.207743:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767857.207747:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099108864:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775767857.207748:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767857.207762:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099109248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767857.207768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767857.207771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767857.207772:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767857.207774:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5318 00000100:00080000:3.0:1775767857.207776:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767857.207779:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767857.207782:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767857.207793:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099109376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767857.207866:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767857.207920:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767857/real 1775767857] req@ffff8b23864f0700 x1862022099108864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767857.207941:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099108864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767857.207954:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767857.207958:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767857.207963:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099108864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767857.207973:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099108992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767857.207999:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767857.208013:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767857/real 1775767857] req@ffff8b23864f3100 x1862022099108992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767857.208025:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099108992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767857.208041:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767857.208044:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767857.208048:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099108992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767857.208077:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099109120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767857.208098:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767857.208120:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767857/real 1775767857] req@ffff8b23864f2d80 x1862022099109120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767857.208133:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099109120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767857.208141:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767857.208176:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767857.208180:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099109120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767857.208187:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099109248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767857.208210:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767857.208222:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767857/real 1775767857] req@ffff8b23864f2680 x1862022099109248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767857.208233:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099109248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767857.208241:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767857.208244:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767857.208248:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099109248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767857.208254:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099109376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767857.208271:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767857.208280:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767857/real 1775767857] req@ffff8b23864f0380 x1862022099109376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767857.208290:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099109376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767857.208297:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767857.208300:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767857.208303:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099109376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767858.103684:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767859.127648:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767860.151645:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767861.175674:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767862.199711:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767862.327598:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767862.327609:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767862.327614:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767862.327626:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5323 00000100:00080000:3.0:1775767862.327649:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767862.327656:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767862.327705:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099109504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767862.327739:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767862.327743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767862.327745:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767862.327750:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5323 00000100:00080000:3.0:1775767862.327754:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767862.327758:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767862.327771:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767862.327789:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099109632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767862.327800:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099109504:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767862.327805:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767862.327810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767862.327813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767862.327818:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5323 00000100:00080000:3.0:1775767862.327823:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767862.327827:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767862.327834:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767862.327850:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099109760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767862.327860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767862.327864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767862.327866:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767862.327870:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5323 00000100:00080000:3.0:1775767862.327874:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767862.327877:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767862.327883:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767862.327891:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767862.327896:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099109888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767862.327907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767862.327911:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767862.327913:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767862.327918:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5323 00000100:00080000:3.0:1775767862.327922:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767862.327926:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767862.327931:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767862.327945:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099110016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767862.327949:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767862/real 1775767862] req@ffff8b23864f0a80 x1862022099109504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767862.327979:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099109504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767862.327990:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767862.328005:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767862.328010:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099109504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767862.328039:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099109632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767862.328063:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767862.328079:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767862/real 1775767862] req@ffff8b23864f1f80 x1862022099109632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767862.328091:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099109632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767862.328113:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767862.328116:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767862.328120:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099109632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767862.328127:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099109760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767862.328149:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767862.328160:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767862/real 1775767862] req@ffff8b23864f1c00 x1862022099109760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767862.328172:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099109760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767862.328181:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767862.328184:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767862.328188:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099109760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767862.328194:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099109888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767862.328214:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767862.328224:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767862/real 1775767862] req@ffff8b23864f0380 x1862022099109888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767862.328236:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099109888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767862.328245:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767862.328249:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767862.328252:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099109888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767862.328258:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099110016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767862.328283:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767862.328291:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767862/real 1775767862] req@ffff8b23864f2680 x1862022099110016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767862.328299:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099110016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767862.328305:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767862.328308:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767862.328310:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099110016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767863.223595:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767864.247632:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767865.271661:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767866.295638:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767867.319659:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767867.447564:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767867.447575:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767867.447580:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767867.447592:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5328 00000100:00080000:3.0:1775767867.447604:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767867.447611:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767867.447657:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099110144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767867.447686:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767867.447689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767867.447690:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767867.447693:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5328 00000100:00080000:3.0:1775767867.447696:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767867.447698:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767867.447708:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767867.447719:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099110272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767867.447736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767867.447740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775767867.447741:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099110144:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775767867.447743:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767867.447752:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5328 00000100:00080000:3.0:1775767867.447755:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767867.447757:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767867.447761:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767867.447770:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099110400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767867.447777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767867.447780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767867.447782:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767867.447785:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5328 00000100:00080000:3.0:1775767867.447787:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767867.447789:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767867.447793:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767867.447800:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099110528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767867.447807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767867.447809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767867.447811:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767867.447813:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5328 00000100:00080000:3.0:1775767867.447815:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767867.447820:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775767867.447821:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:3.0:1775767867.447825:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767867.447834:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099110656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767867.447866:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767867/real 1775767867] req@ffff8b23864f2d80 x1862022099110144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767867.447883:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099110144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767867.447892:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767867.447896:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767867.447899:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099110144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767867.447922:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099110272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767867.447937:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767867.447946:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767867/real 1775767867] req@ffff8b23864f3100 x1862022099110272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767867.447954:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099110272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767867.447959:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767867.447960:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767867.447962:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099110272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767867.447966:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099110400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767867.447977:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767867.447983:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767867/real 1775767867] req@ffff8b23864f0700 x1862022099110400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767867.447990:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099110400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767867.447995:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767867.447996:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767867.447998:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099110400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767867.448001:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099110528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767867.448014:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767867.448019:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767867/real 1775767867] req@ffff8b23864f1180 x1862022099110528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767867.448027:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099110528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767867.448032:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767867.448033:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767867.448035:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099110528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767867.448038:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099110656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767867.448067:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767867.448075:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767867/real 1775767867] req@ffff8b23864f3b80 x1862022099110656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767867.448082:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099110656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767867.448088:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767867.448090:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767867.448092:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099110656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767868.343629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767869.367691:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767870.391618:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767871.415587:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767872.439620:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767872.567553:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767872.567563:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767872.567568:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767872.567579:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5333 00000100:00080000:3.0:1775767872.567590:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767872.567597:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767872.567647:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099110784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767872.567667:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767872.567670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767872.567672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767872.567674:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5333 00000100:00080000:3.0:1775767872.567678:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767872.567680:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767872.567688:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767872.567698:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099110912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767872.567705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767872.567708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767872.567710:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767872.567713:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5333 00000100:00080000:3.0:1775767872.567716:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767872.567719:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767872.567722:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767872.567732:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099110784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767872.567732:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099111040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767872.567746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767872.567748:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767872.567749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767872.567752:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5333 00000100:00080000:3.0:1775767872.567756:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767872.567758:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767872.567760:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767872.567769:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099111168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767872.567779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767872.567783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767872.567785:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767872.567789:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5333 00000100:00080000:3.0:1775767872.567793:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767872.567796:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767872.567801:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767872.567809:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767872.567815:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099111296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767872.567856:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767872/real 1775767872] req@ffff8b23864f0000 x1862022099110784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767872.567871:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099110784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767872.567885:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767872.567889:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767872.567894:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099110784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767872.567902:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099110912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767872.567926:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767872.567940:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767872/real 1775767872] req@ffff8b23864f2a00 x1862022099110912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767872.567952:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099110912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767872.567958:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767872.567960:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767872.567962:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099110912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767872.567980:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099111040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767872.568019:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767872.568029:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767872/real 1775767872] req@ffff8b23864f3480 x1862022099111040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767872.568037:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099111040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767872.568042:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767872.568044:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767872.568047:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099111040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767872.568051:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099111168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767872.568065:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767872.568071:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767872/real 1775767872] req@ffff8b23864f3b80 x1862022099111168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767872.568078:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099111168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767872.568083:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767872.568085:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767872.568087:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099111168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767872.568091:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099111296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767872.568101:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767872.568107:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767872/real 1775767872] req@ffff8b23864f1180 x1862022099111296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767872.568114:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099111296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767872.568118:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767872.568120:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767872.568122:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099111296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767873.463610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767874.487575:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767875.511589:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767876.535669:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767877.559611:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767877.687647:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767877.687657:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767877.687662:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767877.687674:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5339 00000100:00080000:0.0:1775767877.687687:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767877.687693:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767877.687739:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099111424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767877.687767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767877.687770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767877.687771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767877.687775:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5339 00000100:00080000:0.0:1775767877.687777:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767877.687781:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767877.687790:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767877.687803:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099111552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767877.687803:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099111424:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767877.687819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767877.687822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767877.687823:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767877.687825:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5339 00000100:00080000:0.0:1775767877.687828:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767877.687832:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767877.687837:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767877.687851:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099111680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767877.687868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767877.687873:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767877.687875:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767877.687879:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5339 00000800:00000100:3.0:1775767877.687884:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775767877.687886:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767877.687893:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767877.687899:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767877.687912:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099111808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767877.687919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767877.687922:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767877.687923:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767877.687926:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5339 00000100:00080000:0.0:1775767877.687930:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767877.687934:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767877/real 1775767877] req@ffff8b2382dae680 x1862022099111424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767877.687935:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767877.687941:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767877.687978:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099111936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767877.687996:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099111424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767877.688009:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767877.688015:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767877.688021:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099111424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767877.688053:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099111552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767877.688073:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767877.688083:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767877/real 1775767877] req@ffff8b2382dad500 x1862022099111552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767877.688092:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099111552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767877.688098:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767877.688100:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767877.688103:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099111552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767877.688109:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099111680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767877.688121:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767877.688127:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767877/real 1775767877] req@ffff8b2382daea00 x1862022099111680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767877.688135:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099111680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767877.688140:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767877.688142:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767877.688144:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099111680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767877.688147:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099111808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767877.688158:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767877.688164:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767877/real 1775767877] req@ffff8b2382daed80 x1862022099111808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767877.688171:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099111808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767877.688176:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767877.688178:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767877.688180:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099111808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767877.688183:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099111936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767877.688192:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767877.688197:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767877/real 1775767877] req@ffff8b2382daf480 x1862022099111936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767877.688205:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099111936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767877.688210:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767877.688212:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767877.688214:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099111936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767878.583678:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767879.607669:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767880.631794:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767881.655678:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767882.679678:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767882.807612:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767882.807626:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767882.807631:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767882.807645:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5344 00000100:00080000:0.0:1775767882.807662:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767882.807671:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767882.807724:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022099112064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767882.807760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767882.807764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767882.807766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767882.807771:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5344 00000100:00080000:0.0:1775767882.807774:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767882.807778:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767882.807792:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767882.807805:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022099112192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767882.807818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767882.807822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767882.807824:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767882.807830:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5344 00000100:00080000:0.0:1775767882.807835:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767882.807835:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099112064:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767882.807845:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767882.807851:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767882.807865:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022099112320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767882.807877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767882.807881:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767882.807883:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767882.807888:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5344 00000100:00080000:0.0:1775767882.807892:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767882.807917:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767882.807924:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767882.807938:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767882.807946:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022099112448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767882.807957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767882.807961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767882.807963:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767882.807968:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5344 00000100:00080000:0.0:1775767882.807972:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767882.807976:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767882.807982:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767882.807998:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099112576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767882.808000:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767882/real 1775767882] req@ffff8b2382dafb80 x1862022099112064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767882.808036:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022099112064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767882.808046:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767882.808050:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767882.808054:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099112064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767882.808060:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099112192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767882.808083:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767882.808096:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767882/real 1775767882] req@ffff8b2382dadf80 x1862022099112192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767882.808107:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022099112192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767882.808114:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767882.808118:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767882.808121:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099112192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767882.808150:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099112320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767882.808162:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767882.808168:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767882/real 1775767882] req@ffff8b2382dace00 x1862022099112320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767882.808176:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022099112320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767882.808181:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767882.808184:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767882.808186:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099112320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767882.808190:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099112448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767882.808200:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767882.808206:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767882/real 1775767882] req@ffff8b2382dac700 x1862022099112448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767882.808217:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022099112448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767882.808225:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767882.808229:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767882.808232:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099112448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767882.808239:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099112576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767882.808260:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767882.808272:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767882/real 1775767882] req@ffff8b2382daf480 x1862022099112576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767882.808285:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099112576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767882.808294:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767882.808296:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767882.808298:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099112576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767883.703647:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767884.727632:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767885.751577:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767886.775624:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767887.799611:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767887.927578:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767887.927586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767887.927590:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767887.927598:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5349 00000100:00080000:0.0:1775767887.927609:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767887.927615:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767887.927649:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099112704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767887.927672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767887.927677:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767887.927679:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767887.927682:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5349 00000100:00080000:0.0:1775767887.927686:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767887.927689:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767887.927697:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767887.927711:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099112832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767887.927723:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767887.927728:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767887.927730:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767887.927736:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5349 00000100:00080000:0.0:1775767887.927741:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767887.927747:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767887.927754:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767887.927767:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099112704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767887.927772:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099112960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767887.927789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767887.927791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767887.927793:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767887.927797:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5349 00000100:00080000:0.0:1775767887.927801:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767887.927806:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767887.927810:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767887.927819:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099113088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767887.927826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767887.927828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767887.927830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767887.927832:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5349 00000100:00080000:0.0:1775767887.927837:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767887.927841:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767887.927858:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767887.927871:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099113216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767887.927885:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767887.927942:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767887/real 1775767887] req@ffff8b2382daed80 x1862022099112704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767887.927968:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099112704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767887.927980:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767887.927986:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767887.927989:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099112704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767887.928004:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099112832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767887.928028:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767887.928039:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767887/real 1775767887] req@ffff8b2382daea00 x1862022099112832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767887.928047:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099112832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767887.928052:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767887.928054:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767887.928056:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099112832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767887.928083:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099112960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767887.928094:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767887.928100:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767887/real 1775767887] req@ffff8b2382dad500 x1862022099112960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767887.928108:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099112960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767887.928114:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767887.928116:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767887.928118:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099112960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767887.928122:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099113088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767887.928133:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767887.928138:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767887/real 1775767887] req@ffff8b2382dae680 x1862022099113088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767887.928146:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099113088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767887.928151:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767887.928153:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767887.928155:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099113088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767887.928158:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099113216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767887.928168:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767887.928173:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767887/real 1775767887] req@ffff8b2382daca80 x1862022099113216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767887.928179:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099113216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767887.928185:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767887.928187:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767887.928189:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099113216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767888.823659:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767889.847568:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767890.871604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767891.895682:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767892.919632:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767893.047558:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767893.047566:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767893.047571:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767893.047581:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5354 00000100:00080000:0.0:1775767893.047593:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767893.047600:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767893.047641:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022099113344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767893.047677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767893.047682:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767893.047683:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767893.047686:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5354 00000100:00080000:0.0:1775767893.047690:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767893.047693:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767893.047701:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767893.047713:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099113472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767893.047723:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767893.047725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767893.047727:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767893.047732:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099113344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767893.047734:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5354 00000100:00080000:0.0:1775767893.047740:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767893.047743:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767893.047746:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767893.047756:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022099113600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767893.047762:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767893.047765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767893.047766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767893.047769:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5354 00000100:00080000:0.0:1775767893.047776:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767893.047778:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767893.047782:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767893.047794:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022099113728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767893.047816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1775767893.047817:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775767893.047821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767893.047823:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767893.047827:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5354 00000100:00080000:0.0:1775767893.047831:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767893.047835:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767893.047840:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767893.047849:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099113856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767893.047862:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767893/real 1775767893] req@ffff8b2382dad880 x1862022099113344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767893.047879:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022099113344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767893.047888:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767893.047895:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767893.047903:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099113344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767893.047929:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099113472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767893.047955:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767893.047969:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767893/real 1775767893] req@ffff8b2382dac380 x1862022099113472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767893.047982:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099113472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767893.047987:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767893.047989:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767893.047992:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099113472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767893.047997:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099113600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767893.048014:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767893.048022:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767893/real 1775767893] req@ffff8b2382dac000 x1862022099113600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767893.048029:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022099113600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767893.048035:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767893.048036:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767893.048038:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099113600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767893.048043:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099113728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767893.048054:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767893.048059:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767893/real 1775767893] req@ffff8b2382daf100 x1862022099113728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767893.048066:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022099113728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767893.048071:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767893.048073:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767893.048074:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099113728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767893.048077:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099113856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767893.048087:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767893.048091:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767893/real 1775767893] req@ffff8b2382daca80 x1862022099113856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767893.048098:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099113856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767893.048103:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767893.048105:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767893.048106:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099113856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767893.943643:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767894.967610:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767895.991704:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767897.015639:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767898.039641:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767898.167557:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767898.167570:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767898.167577:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767898.167588:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5359 00000100:00080000:0.0:1775767898.167601:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767898.167608:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767898.167678:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099113984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767898.167702:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767898.167705:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767898.167707:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767898.167709:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5359 00000100:00080000:0.0:1775767898.167713:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767898.167716:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767898.167726:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767898.167736:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099114112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767898.167761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767898.167764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767898.167767:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767898.167768:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099113984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767898.167773:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5359 00000100:00080000:0.0:1775767898.167779:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767898.167782:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767898.167787:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767898.167799:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099114240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767898.167807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767898.167810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767898.167811:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767898.167814:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5359 00000100:00080000:0.0:1775767898.167817:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767898.167820:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767898.167823:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767898.167830:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099114368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767898.167841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767898.167843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767898.167847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775767898.167847:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775767898.167851:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5359 00000100:00080000:0.0:1775767898.167854:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767898.167856:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767898.167859:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767898.167872:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099114496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767898.167897:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767898/real 1775767898] req@ffff8b2382dae680 x1862022099113984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767898.167912:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099113984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767898.167921:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767898.167924:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767898.167927:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099113984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767898.167950:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099114112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767898.167965:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767898.167974:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767898/real 1775767898] req@ffff8b2382dad500 x1862022099114112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767898.167987:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099114112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767898.167995:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767898.167999:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767898.168002:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099114112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767898.168009:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099114240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767898.168031:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767898.168042:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767898/real 1775767898] req@ffff8b2382daea00 x1862022099114240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767898.168056:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099114240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767898.168062:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767898.168064:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767898.168066:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099114240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767898.168071:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099114368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767898.168091:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767898.168112:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767898/real 1775767898] req@ffff8b2382daed80 x1862022099114368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767898.168126:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099114368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767898.168135:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767898.168137:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767898.168140:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099114368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767898.168144:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099114496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767898.168161:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767898.168168:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767898/real 1775767898] req@ffff8b2382daf480 x1862022099114496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767898.168175:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099114496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767898.168185:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767898.168187:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767898.168189:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099114496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767899.063676:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767900.087679:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767901.111682:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767902.135662:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767903.159638:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767903.287570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767903.287581:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767903.287586:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767903.287597:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5364 00000100:00080000:0.0:1775767903.287609:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767903.287616:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767903.287657:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022099114624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767903.287688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767903.287712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767903.287714:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767903.287718:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5364 00000100:00080000:0.0:1775767903.287722:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767903.287726:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767903.287736:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767903.287747:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022099114752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767903.287757:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099114624:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775767903.287759:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767903.287766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767903.287769:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767903.287773:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5364 00000100:00080000:0.0:1775767903.287775:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767903.287777:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767903.287780:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767903.287790:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022099114880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767903.287797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767903.287800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767903.287801:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767903.287804:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5364 00000100:00080000:0.0:1775767903.287807:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767903.287809:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767903.287812:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767903.287819:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022099115008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767903.287825:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767903.287828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767903.287829:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767903.287832:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5364 00000100:00080000:0.0:1775767903.287835:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775767903.287838:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775767903.287841:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767903.287845:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767903.287852:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099115136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767903.287889:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767903/real 1775767903] req@ffff8b2382dac700 x1862022099114624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767903.287910:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022099114624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767903.287920:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767903.287925:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767903.287929:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099114624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767903.287954:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099114752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767903.287979:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767903.287993:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767903/real 1775767903] req@ffff8b2382dace00 x1862022099114752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767903.288006:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022099114752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767903.288015:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767903.288020:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767903.288024:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099114752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767903.288030:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099114880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767903.288050:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767903.288060:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767903/real 1775767903] req@ffff8b2382dadf80 x1862022099114880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767903.288072:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022099114880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767903.288080:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767903.288084:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767903.288088:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099114880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767903.288094:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099115008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767903.288131:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767903.288143:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767903/real 1775767903] req@ffff8b2382dafb80 x1862022099115008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767903.288156:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022099115008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767903.288165:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767903.288168:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767903.288172:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099115008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767903.288179:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099115136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767903.288198:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767903.288208:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767903/real 1775767903] req@ffff8b2382daf480 x1862022099115136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767903.288220:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099115136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767903.288228:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767903.288231:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767903.288233:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099115136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767904.183641:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767905.207636:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767906.231697:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767907.255651:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767908.279631:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767908.407638:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767908.407661:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767908.407669:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767908.407682:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5369 00000100:00080000:1.0:1775767908.407698:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767908.407707:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767908.407765:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099115264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767908.407795:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767908.407799:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767908.407801:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767908.407804:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5369 00000100:00080000:1.0:1775767908.407808:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767908.407811:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767908.407821:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767908.407833:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099115392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767908.407843:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767908.407846:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767908.407847:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767908.407853:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099115264:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767908.407854:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5369 00000100:00080000:1.0:1775767908.407861:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767908.407865:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767908.407868:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767908.407877:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099115520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767908.407883:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767908.407886:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767908.407887:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767908.407891:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5369 00000100:00080000:1.0:1775767908.407893:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767908.407895:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767908.407899:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767908.407905:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099115648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767908.407911:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767908.407914:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767908.407915:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767908.407918:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5369 00000100:00080000:1.0:1775767908.407922:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767908.407924:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767908.407926:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767908.407932:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775767908.407939:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099115776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767908.407986:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767908/real 1775767908] req@ffff8b23864ffb80 x1862022099115264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767908.408004:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099115264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767908.408014:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767908.408018:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767908.408021:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099115264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767908.408047:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099115392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767908.408064:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767908.408074:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767908/real 1775767908] req@ffff8b23864fe300 x1862022099115392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767908.408083:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099115392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767908.408088:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767908.408091:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767908.408093:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099115392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767908.408098:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099115520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767908.408108:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767908.408114:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767908/real 1775767908] req@ffff8b23864ff100 x1862022099115520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767908.408122:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099115520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767908.408127:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767908.408129:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767908.408131:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099115520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767908.408135:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099115648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767908.408146:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767908.408151:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767908/real 1775767908] req@ffff8b23864fd500 x1862022099115648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767908.408159:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099115648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767908.408164:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767908.408167:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767908.408169:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099115648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767908.408172:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099115776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767908.408181:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767908.408186:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767908/real 1775767908] req@ffff8b23864fdc00 x1862022099115776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767908.408193:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099115776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767908.408198:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767908.408200:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767908.408202:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099115776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767909.303697:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767910.327671:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767911.351713:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767912.375663:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767913.399640:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767913.527559:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767913.527569:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767913.527574:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767913.527585:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5374 00000100:00080000:2.0:1775767913.527611:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767913.527618:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767913.527664:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099115904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767913.527690:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767913.527694:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767913.527696:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767913.527701:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5374 00000100:00080000:2.0:1775767913.527704:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767913.527707:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767913.527717:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767913.527732:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099116032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767913.527747:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767913.527752:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1775767913.527753:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099115904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767913.527754:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767913.527775:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5374 00000100:00080000:2.0:1775767913.527780:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767913.527785:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767913.527790:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767913.527803:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099116160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767913.527811:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767913.527814:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767913.527815:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767913.527819:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5374 00000100:00080000:2.0:1775767913.527824:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767913.527826:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767913.527830:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767913.527837:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099116288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767913.527843:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767913.527846:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767913.527847:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767913.527849:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5374 00000100:00080000:2.0:1775767913.527852:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767913.527854:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767913.527856:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767913.527866:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099116416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767913.527909:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099116032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767913.527944:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099116160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767913.527971:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099116288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767913.527993:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099116416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767914.423618:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767915.447756:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767916.471635:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767916.599871:0:2365:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775767916.599943:0:2365:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767916.599967:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767916.600018:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767916.600030:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767916.600041:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775767916.600051:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:3.0:1775767916.600089:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767913/real 1775767916] req@ffff8b23b9ea7480 x1862022099116032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767916.600126:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099116032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767916.600140:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767916.600146:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767916.600152:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099116032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767916.600163:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767913/real 1775767916] req@ffff8b23b9ea6300 x1862022099116160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767916.600176:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099116160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767916.600185:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767916.600189:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767916.600193:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099116160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767916.600200:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767913/real 1775767916] req@ffff8b23b9ea7100 x1862022099116288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767916.600211:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099116288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767916.600219:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767916.600224:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767916.600227:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099116288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767916.600234:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767913/real 1775767916] req@ffff8b23b9ea4380 x1862022099116416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767916.600246:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099116416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767916.600254:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767916.600258:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767916.600261:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099116416:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767916.600268:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767913/real 1775767916] req@ffff8b23b9ea4700 x1862022099115904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767916.600280:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099115904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767916.600289:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767916.600292:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767916.600295:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099115904:192.168.202.128@tcp:250:kworker.0 00010000:00010000:1.0:1775767917.495689:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767918.519690:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767918.647573:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767918.647584:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767918.647587:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767918.647596:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5380 00000100:00080000:2.0:1775767918.647607:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767918.647613:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767918.647643:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022099116544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767918.647671:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767918.647676:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767918.647677:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767918.647680:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5380 00000100:00080000:2.0:1775767918.647682:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767918.647685:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767918.647692:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767918.647701:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099116672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767918.647710:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767918.647718:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767918.647720:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767918.647724:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099116544:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767918.647727:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5380 00000100:00080000:2.0:1775767918.647732:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767918.647735:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767918.647739:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767918.647748:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099116800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767918.647754:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767918.647757:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767918.647758:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767918.647761:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5380 00000100:00080000:2.0:1775767918.647764:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767918.647767:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767918.647771:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767918.647777:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099116928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767918.647784:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767918.647787:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767918.647788:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767918.647790:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5380 00000100:00080000:2.0:1775767918.647792:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767918.647794:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767918.647797:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767918.647805:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775767918.647806:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099117056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767918.647865:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767918/real 1775767918] req@ffff8b23b9ea4e00 x1862022099116544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767918.647889:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022099116544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767918.647904:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767918.647909:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767918.647914:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099116544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767918.647924:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099116672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767918.647949:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767918.647964:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767918/real 1775767918] req@ffff8b23b9ea4700 x1862022099116672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767918.647977:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099116672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767918.647994:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767918.647997:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767918.648000:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099116672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767918.648023:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099116800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767918.648045:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767918.648056:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767918/real 1775767918] req@ffff8b23b9ea4380 x1862022099116800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767918.648068:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099116800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767918.648077:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767918.648080:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767918.648083:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099116800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767918.648090:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099116928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767918.648113:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767918.648124:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767918/real 1775767918] req@ffff8b23b9ea7100 x1862022099116928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767918.648136:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099116928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767918.648144:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767918.648147:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767918.648150:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099116928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767918.648155:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099117056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767918.648174:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767918.648184:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767918/real 1775767918] req@ffff8b23b9ea6300 x1862022099117056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767918.648196:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099117056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767918.648203:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767918.648206:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767918.648209:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099117056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767919.543642:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767920.567645:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767921.591660:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767922.615618:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767923.639678:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767923.767575:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767923.767583:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767923.767587:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767923.767596:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5385 00000100:00080000:2.0:1775767923.767605:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767923.767609:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767923.767650:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099117184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767923.767673:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767923.767677:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767923.767679:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767923.767683:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5385 00000100:00080000:2.0:1775767923.767688:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767923.767693:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767923.767704:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767923.767718:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099117184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775767923.767719:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099117312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767923.767740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767923.767744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767923.767746:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767923.767750:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5385 00000100:00080000:2.0:1775767923.767755:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767923.767759:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767923.767765:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767923.767778:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099117440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767923.767786:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775767923.767788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767923.767791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767923.767792:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767923.767797:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5385 00000100:00080000:2.0:1775767923.767801:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767923.767804:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767923.767808:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767923.767816:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099117568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767923.767824:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767923/real 1775767923] req@ffff8b23b9ea7480 x1862022099117184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767923.767835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767923.767843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767923.767845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767923.767849:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5385 00000100:00080000:2.0:1775767923.767852:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1775767923.767855:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099117184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767923.767858:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767923.767864:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1775767923.767870:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1775767923.767878:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099117696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767923.767878:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767923.767889:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099117184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767923.767917:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099117312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767923.767941:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767923.767961:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767923/real 1775767923] req@ffff8b23b9ea5880 x1862022099117312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767923.767970:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099117312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767923.767976:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767923.767978:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767923.767981:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099117312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767923.767985:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099117440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767923.767998:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767923.768004:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767923/real 1775767923] req@ffff8b23b9ea4000 x1862022099117440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767923.768015:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099117440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767923.768020:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767923.768022:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767923.768024:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099117440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767923.768028:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099117568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767923.768038:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767923.768043:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767923/real 1775767923] req@ffff8b23b9ea6a00 x1862022099117568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767923.768051:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099117568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767923.768055:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767923.768057:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767923.768059:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099117568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767923.768062:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099117696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767923.768071:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767923.768076:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767923/real 1775767923] req@ffff8b23b9ea4a80 x1862022099117696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767923.768082:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099117696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767923.768098:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767923.768101:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767923.768103:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099117696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767924.663649:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767925.687655:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767926.711643:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767927.735629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767928.759613:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767928.887546:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767928.887554:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767928.887557:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767928.887567:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5390 00000100:00080000:2.0:1775767928.887577:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767928.887582:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767928.887614:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099117824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767928.887639:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767928.887642:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767928.887643:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767928.887646:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5390 00000100:00080000:2.0:1775767928.887648:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767928.887650:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767928.887665:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767928.887674:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099117952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767928.887683:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767928.887686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767928.887687:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767928.887691:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5390 00000100:00080000:2.0:1775767928.887693:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767928.887696:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767928.887700:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767928.887716:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099118080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767928.887718:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099117824:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767928.887739:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767928.887742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767928.887743:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767928.887746:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5390 00000100:00080000:2.0:1775767928.887748:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767928.887750:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767928.887755:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767928.887764:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099118208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767928.887770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767928.887771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767928.887773:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767928.887776:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5390 00000100:00080000:2.0:1775767928.887778:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767928.887780:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767928.887783:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767928.887789:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099118336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767928.887831:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767928.887882:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767928/real 1775767928] req@ffff8b23b9ea7b80 x1862022099117824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767928.887905:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099117824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767928.887918:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767928.887933:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767928.887938:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099117824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767928.887948:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099117952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767928.887974:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767928.887987:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767928/real 1775767928] req@ffff8b23b9ea6680 x1862022099117952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767928.887996:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099117952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767928.888007:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767928.888009:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767928.888012:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099117952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767928.888036:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099118080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767928.888049:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767928.888055:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767928/real 1775767928] req@ffff8b23b9ea5f80 x1862022099118080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767928.888063:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099118080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767928.888068:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767928.888071:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767928.888073:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099118080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767928.888077:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099118208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767928.888087:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767928.888092:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767928/real 1775767928] req@ffff8b23b9ea6d80 x1862022099118208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767928.888099:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099118208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767928.888104:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767928.888106:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767928.888108:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099118208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767928.888112:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099118336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767928.888121:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767928.888126:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767928/real 1775767928] req@ffff8b23b9ea5c00 x1862022099118336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767928.888132:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099118336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767928.888137:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767928.888139:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767928.888141:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099118336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767929.783616:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767930.807654:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767931.831667:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767932.855669:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767933.879608:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767934.007556:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767934.007564:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767934.007569:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767934.007580:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5395 00000100:00080000:2.0:1775767934.007592:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767934.007599:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767934.007638:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022099118464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767934.007667:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767934.007670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767934.007672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767934.007674:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5395 00000100:00080000:2.0:1775767934.007678:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767934.007680:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767934.007688:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767934.007699:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099118592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767934.007713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767934.007715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767934.007717:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767934.007720:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5395 00000100:00080000:2.0:1775767934.007723:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767934.007728:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767934.007730:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099118464:192.168.202.128@tcp:250:kworker.0 00000008:00100000:2.0:1775767934.007731:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767934.007741:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099118720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767934.007747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767934.007750:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767934.007751:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767934.007754:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5395 00000100:00080000:2.0:1775767934.007756:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767934.007757:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767934.007761:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767934.007769:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099118848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767934.007775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767934.007778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767934.007779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767934.007781:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5395 00000100:00080000:2.0:1775767934.007784:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767934.007786:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775767934.007791:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:2.0:1775767934.007792:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767934.007807:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099118976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767934.007836:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767934/real 1775767934] req@ffff8b23b9ea5500 x1862022099118464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767934.007855:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022099118464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767934.007868:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767934.007873:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767934.007877:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099118464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767934.007892:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099118592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767934.007915:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767934.007924:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767934/real 1775767934] req@ffff8b23b9ea5c00 x1862022099118592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767934.007933:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099118592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767934.007939:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767934.007941:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767934.007944:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099118592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767934.007963:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099118720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767934.007975:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767934.007981:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767934/real 1775767934] req@ffff8b23b9ea6d80 x1862022099118720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767934.007993:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099118720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767934.008002:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767934.008005:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767934.008009:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099118720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767934.008015:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099118848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767934.008035:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767934.008044:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767934/real 1775767934] req@ffff8b23b9ea5f80 x1862022099118848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767934.008057:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099118848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767934.008063:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767934.008065:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767934.008067:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099118848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767934.008071:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099118976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767934.008082:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767934.008087:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767934/real 1775767934] req@ffff8b23b9ea6680 x1862022099118976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767934.008094:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099118976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767934.008099:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767934.008101:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767934.008102:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099118976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767934.903668:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767935.927648:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767936.951703:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767937.975658:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767938.999696:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767939.127611:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767939.127622:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767939.127627:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767939.127637:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5400 00000100:00080000:1.0:1775767939.127649:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767939.127656:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767939.127707:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099119104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767939.127744:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767939.127747:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767939.127753:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767939.127757:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5400 00000100:00080000:1.0:1775767939.127760:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767939.127763:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767939.127775:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767939.127795:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099119232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767939.127810:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099119104:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767939.127811:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767939.127821:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767939.127823:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767939.127829:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5400 00000100:00080000:1.0:1775767939.127833:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767939.127838:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767939.127845:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767939.127858:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099119360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767939.127866:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767939.127869:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767939.127870:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767939.127874:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5400 00000100:00080000:1.0:1775767939.127877:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767939.127880:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767939.127885:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767939.127893:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099119488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767939.127900:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1775767939.127902:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775767939.127905:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767939.127908:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767939.127911:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5400 00000100:00080000:1.0:1775767939.127913:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767939.127916:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767939.127919:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767939.127930:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099119616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767939.127970:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767939/real 1775767939] req@ffff8b23864fe680 x1862022099119104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767939.127995:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099119104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767939.128004:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767939.128008:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767939.128011:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099119104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767939.128035:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099119232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767939.128053:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767939.128062:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767939/real 1775767939] req@ffff8b23864fca80 x1862022099119232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767939.128070:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099119232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767939.128076:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767939.128078:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767939.128080:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099119232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767939.128084:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099119360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767939.128094:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767939.128099:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767939/real 1775767939] req@ffff8b23864fea00 x1862022099119360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767939.128107:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099119360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767939.128116:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767939.128118:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767939.128121:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099119360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767939.128124:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099119488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767939.128134:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767939.128139:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767939/real 1775767939] req@ffff8b23864fd180 x1862022099119488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767939.128146:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099119488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767939.128152:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767939.128153:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767939.128155:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099119488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767939.128158:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099119616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767939.128168:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767939.128172:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767939/real 1775767939] req@ffff8b23864fce00 x1862022099119616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767939.128179:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099119616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767939.128184:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767939.128186:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767939.128188:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099119616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767940.023635:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767941.047685:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767942.071790:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767943.095624:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767944.119601:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767944.247525:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767944.247536:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767944.247541:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767944.247552:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5405 00000100:00080000:1.0:1775767944.247565:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767944.247572:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767944.247618:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099119744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767944.247645:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767944.247648:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767944.247650:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767944.247652:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5405 00000100:00080000:1.0:1775767944.247655:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767944.247659:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767944.247671:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767944.247688:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099119872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767944.247701:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767944.247705:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767944.247707:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767944.247714:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5405 00000100:00100000:3.0:1775767944.247715:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099119744:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767944.247718:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767944.247726:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767944.247731:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767944.247743:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099120000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767944.247752:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767944.247754:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767944.247756:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767944.247760:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5405 00000100:00080000:1.0:1775767944.247764:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767944.247766:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767944.247771:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767944.247781:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099120128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767944.247792:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767944.247795:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767944.247797:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767944.247803:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5405 00000800:00000100:3.0:1775767944.247804:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775767944.247810:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767944.247814:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767944.247820:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767944.247834:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099120256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767944.247864:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767944/real 1775767944] req@ffff8b23864fdf80 x1862022099119744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767944.247883:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099119744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767944.247893:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767944.247896:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767944.247899:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099119744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767944.247905:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099119872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767944.247931:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767944.247940:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767944/real 1775767944] req@ffff8b23864ff800 x1862022099119872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767944.247948:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099119872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767944.247954:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767944.247956:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767944.247959:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099119872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767944.247982:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099120000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767944.247994:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767944.248000:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767944/real 1775767944] req@ffff8b23864fed80 x1862022099120000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767944.248008:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099120000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767944.248013:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767944.248015:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767944.248017:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099120000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767944.248021:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099120128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767944.248032:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767944.248037:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767944/real 1775767944] req@ffff8b23864fc000 x1862022099120128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767944.248045:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099120128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767944.248049:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767944.248051:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767944.248053:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099120128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767944.248056:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099120256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767944.248066:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767944.248070:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767944/real 1775767944] req@ffff8b23864fc380 x1862022099120256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767944.248082:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099120256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767944.248087:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767944.248089:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767944.248091:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099120256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767945.143646:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767946.167612:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767947.191633:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767948.215640:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767949.239622:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767949.367614:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767949.367626:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767949.367631:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767949.367643:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5410 00000100:00080000:1.0:1775767949.367657:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767949.367665:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767949.367708:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099120384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767949.367748:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767949.367753:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767949.367755:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767949.367760:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5410 00000100:00080000:1.0:1775767949.367764:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767949.367768:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767949.367777:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767949.367791:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099120512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767949.367803:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767949.367808:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767949.367810:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775767949.367810:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099120384:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775767949.367818:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5410 00000100:00080000:1.0:1775767949.367821:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767949.367824:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767949.367827:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767949.367838:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099120640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767949.367846:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767949.367849:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767949.367851:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767949.367854:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5410 00000100:00080000:1.0:1775767949.367858:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767949.367861:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767949.367865:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767949.367874:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099120768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767949.367883:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775767949.367887:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767949.367890:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767949.367891:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767949.367910:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5410 00000100:00080000:1.0:1775767949.367914:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767949.367916:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767949.367920:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767949.367936:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099120896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767949.367966:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767949/real 1775767949] req@ffff8b23864fc380 x1862022099120384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767949.367987:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099120384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767949.367998:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767949.368001:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767949.368005:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099120384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767949.368026:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099120512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767949.368045:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767949.368054:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767949/real 1775767949] req@ffff8b23864fc000 x1862022099120512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767949.368067:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099120512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767949.368072:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767949.368074:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767949.368076:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099120512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767949.368080:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099120640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767949.368091:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767949.368096:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767949/real 1775767949] req@ffff8b23864fed80 x1862022099120640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767949.368104:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099120640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767949.368109:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767949.368110:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767949.368112:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099120640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767949.368115:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099120768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767949.368136:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767949.368142:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767949/real 1775767949] req@ffff8b23864ff800 x1862022099120768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767949.368149:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099120768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767949.368159:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767949.368161:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767949.368163:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099120768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767949.368166:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099120896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767949.368176:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767949.368181:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767949/real 1775767949] req@ffff8b23864fdf80 x1862022099120896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767949.368187:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099120896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767949.368209:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767949.368211:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767949.368213:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099120896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767950.263622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767951.287612:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767952.311672:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767953.335633:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767954.359673:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767954.487538:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767954.487546:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767954.487550:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767954.487558:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5415 00000100:00080000:0.0:1775767954.487567:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767954.487572:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767954.487606:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099121024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767954.487631:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767954.487634:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767954.487635:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767954.487638:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5415 00000100:00080000:0.0:1775767954.487640:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767954.487642:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767954.487650:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767954.487659:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099121152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767954.487672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767954.487675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767954.487676:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767954.487679:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5415 00000100:00080000:0.0:1775767954.487681:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767954.487686:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767954.487686:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099121024:192.168.202.128@tcp:250:kworker.0 00000008:00100000:0.0:1775767954.487690:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767954.487703:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099121280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767954.487710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767954.487712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767954.487714:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767954.487716:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5415 00000100:00080000:0.0:1775767954.487719:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767954.487722:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767954.487725:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767954.487731:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099121408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767954.487737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767954.487740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767954.487741:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775767954.487746:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775767954.487748:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5415 00000100:00080000:0.0:1775767954.487753:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767954.487756:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767954.487760:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767954.487768:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099121536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767954.487787:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767954/real 1775767954] req@ffff8b2382daed80 x1862022099121024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767954.487801:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099121024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767954.487808:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767954.487812:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767954.487816:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099121024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767954.487824:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099121152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767954.487846:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767954.487860:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767954/real 1775767954] req@ffff8b2382daea00 x1862022099121152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767954.487874:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099121152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767954.487882:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767954.487886:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767954.487889:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099121152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767954.487908:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099121280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767954.487924:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767954.487931:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767954/real 1775767954] req@ffff8b2382dad500 x1862022099121280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767954.487939:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099121280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767954.487944:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767954.487946:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767954.487948:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099121280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767954.487952:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099121408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767954.487963:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767954.487969:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767954/real 1775767954] req@ffff8b2382dae680 x1862022099121408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767954.487976:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099121408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767954.487981:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767954.487982:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767954.487984:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099121408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767954.487987:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099121536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767954.487996:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767954.488001:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767954/real 1775767954] req@ffff8b2382daca80 x1862022099121536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767954.488008:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099121536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767954.488012:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767954.488014:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767954.488016:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099121536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767955.383632:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767956.407612:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767957.431654:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767958.455673:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767959.479638:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767959.607624:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767959.607645:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767959.607652:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767959.607663:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5421 00000100:00080000:2.0:1775767959.607676:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767959.607684:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767959.607729:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099121664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767959.607766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767959.607769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767959.607771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767959.607774:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5421 00000100:00080000:2.0:1775767959.607776:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767959.607778:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767959.607788:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767959.607796:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099121792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767959.607806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767959.607811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767959.607812:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767959.607816:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5421 00000100:00080000:2.0:1775767959.607831:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767959.607833:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767959.607836:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767959.607841:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099121664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775767959.607847:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099121920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767959.607856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767959.607858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767959.607859:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767959.607863:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5421 00000100:00080000:2.0:1775767959.607865:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767959.607867:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767959.607871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767959.607880:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099122048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767959.607886:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767959.607888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767959.607890:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767959.607892:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5421 00000100:00080000:2.0:1775767959.607908:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767959.607911:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767959.607916:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767959.607923:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099122176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775767959.607930:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767959.607979:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767959/real 1775767959] req@ffff8b23b9ea7b80 x1862022099121664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767959.607995:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099121664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767959.608003:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767959.608007:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767959.608011:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099121664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767959.608016:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099121792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767959.608033:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767959.608041:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767959/real 1775767959] req@ffff8b23b9ea4a80 x1862022099121792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767959.608049:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099121792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767959.608054:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767959.608056:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767959.608058:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099121792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767959.608080:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099121920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767959.608091:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767959.608097:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767959/real 1775767959] req@ffff8b23b9ea6a00 x1862022099121920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767959.608105:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099121920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767959.608115:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767959.608117:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767959.608119:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099121920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767959.608123:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099122048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767959.608133:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767959.608138:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767959/real 1775767959] req@ffff8b23b9ea4000 x1862022099122048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767959.608145:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099122048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767959.608150:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767959.608170:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767959.608173:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099122048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767959.608177:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099122176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767959.608203:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767959.608210:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767959/real 1775767959] req@ffff8b23b9ea5880 x1862022099122176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767959.608218:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099122176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767959.608224:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767959.608226:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767959.608228:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099122176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767960.503667:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767961.527622:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767962.551685:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767963.575649:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767964.599679:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767964.727596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767964.727608:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767964.727612:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767964.727623:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5426 00000100:00080000:0.0:1775767964.727636:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767964.727643:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767964.727689:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022099122304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767964.727733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767964.727737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767964.727739:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767964.727742:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5426 00000100:00080000:0.0:1775767964.727745:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767964.727748:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767964.727758:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767964.727768:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022099122432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767964.727798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767964.727805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767964.727807:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767964.727813:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5426 00000100:00080000:0.0:1775767964.727817:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767964.727822:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767964.727829:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767964.727831:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099122304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767964.727846:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099122560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767964.727859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767964.727863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767964.727865:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767964.727871:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5426 00000100:00080000:0.0:1775767964.727876:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767964.727881:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767964.727887:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767964.727902:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022099122688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767964.727911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1775767964.727911:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775767964.727916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767964.727918:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767964.727922:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5426 00000100:00080000:0.0:1775767964.727927:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767964.727931:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767964.727936:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767964.727946:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099122816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767964.727965:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767964/real 1775767964] req@ffff8b2382daf100 x1862022099122304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767964.727988:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022099122304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767964.727999:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767964.728004:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767964.728009:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099122304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767964.728018:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099122432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767964.728043:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767964.728056:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767964/real 1775767964] req@ffff8b2382dac000 x1862022099122432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767964.728065:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022099122432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767964.728070:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767964.728073:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767964.728075:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099122432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767964.728126:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099122560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767964.728143:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767964.728152:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767964/real 1775767964] req@ffff8b2382dac380 x1862022099122560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767964.728160:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099122560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767964.728165:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767964.728168:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767964.728170:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099122560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767964.728174:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099122688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767964.728186:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767964.728191:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767964/real 1775767964] req@ffff8b2382dad880 x1862022099122688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767964.728198:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022099122688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767964.728203:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767964.728205:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767964.728207:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099122688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767964.728210:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099122816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767964.728227:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767964.728234:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767964/real 1775767964] req@ffff8b2382daca80 x1862022099122816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767964.728247:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099122816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767964.728252:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767964.728254:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767964.728256:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099122816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767965.623590:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767966.647639:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767967.671584:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767968.695628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767969.719678:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767969.847594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767969.847605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767969.847610:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767969.847621:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5431 00000100:00080000:0.0:1775767969.847634:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767969.847642:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767969.847705:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099122944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767969.847740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767969.847744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767969.847746:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767969.847750:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5431 00000100:00080000:0.0:1775767969.847754:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767969.847759:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767969.847770:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767969.847786:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099123072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767969.847798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767969.847802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767969.847804:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767969.847808:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5431 00000100:00080000:0.0:1775767969.847813:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767969.847818:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767969.847823:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767969.847825:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099122944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767969.847844:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099123200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767969.847855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767969.847859:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767969.847861:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767969.847865:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5431 00000100:00080000:0.0:1775767969.847869:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767969.847873:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775767969.847878:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767969.847889:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099123328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767969.847897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767969.847901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775767969.847903:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767969.847906:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5431 00000100:00080000:0.0:1775767969.847910:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767969.847915:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:3.0:1775767969.847915:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:0.0:1775767969.847922:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767969.847933:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099123456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767969.847964:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767969/real 1775767969] req@ffff8b2382dae680 x1862022099122944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767969.847980:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099122944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767969.847989:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767969.847993:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767969.847996:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099122944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767969.848001:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099123072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767969.848016:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767969.848024:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767969/real 1775767969] req@ffff8b2382dad500 x1862022099123072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767969.848037:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099123072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767969.848042:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767969.848044:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767969.848046:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099123072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767969.848090:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099123200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767969.848107:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767969.848115:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767969/real 1775767969] req@ffff8b2382daea00 x1862022099123200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767969.848123:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099123200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767969.848129:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767969.848131:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767969.848133:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099123200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767969.848137:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099123328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767969.848148:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767969.848153:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767969/real 1775767969] req@ffff8b2382daed80 x1862022099123328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767969.848160:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099123328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767969.848170:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767969.848172:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767969.848175:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099123328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767969.848178:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099123456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767969.848187:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767969.848192:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767969/real 1775767969] req@ffff8b2382daf480 x1862022099123456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767969.848199:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099123456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767969.848205:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767969.848206:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767969.848208:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099123456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767970.743730:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767971.767645:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767972.791665:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767973.815748:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767974.839634:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767974.967624:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767974.967635:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767974.967640:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767974.967650:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5436 00000100:00080000:1.0:1775767974.967664:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767974.967672:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775767974.967721:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099123584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767974.967749:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767974.967752:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767974.967754:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767974.967757:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5436 00000100:00080000:1.0:1775767974.967760:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767974.967764:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767974.967773:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767974.967780:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099123584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775767974.967787:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099123712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767974.967796:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767974.967799:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767974.967800:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767974.967804:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5436 00000100:00080000:1.0:1775767974.967806:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767974.967809:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767974.967813:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767974.967824:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099123840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767974.967830:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767974.967833:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767974.967834:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767974.967837:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5436 00000100:00080000:1.0:1775767974.967843:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767974.967846:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767974.967849:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775767974.967855:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775767974.967858:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099123968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767974.967866:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767974.967869:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775767974.967870:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767974.967874:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5436 00000100:00080000:1.0:1775767974.967883:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767974.967885:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775767974.967888:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767974.967895:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099124096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775767974.967925:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767974/real 1775767974] req@ffff8b23864fce00 x1862022099123584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767974.967949:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099123584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767974.967960:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767974.967965:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767974.967969:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099123584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775767974.967995:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099123712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767974.968037:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767974.968052:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767974/real 1775767974] req@ffff8b23864fd180 x1862022099123712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767974.968066:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099123712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767974.968072:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767974.968074:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767974.968077:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099123712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767974.968082:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099123840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775767974.968099:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767974.968106:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767974/real 1775767974] req@ffff8b23864fea00 x1862022099123840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767974.968114:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099123840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767974.968119:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767974.968121:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767974.968123:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099123840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775767974.968127:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099123968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767974.968140:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767974.968146:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767974/real 1775767974] req@ffff8b23864fca80 x1862022099123968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767974.968153:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099123968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767974.968167:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767974.968170:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767974.968171:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099123968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775767974.968176:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099124096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775767974.968187:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775767974.968192:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767974/real 1775767974] req@ffff8b23864fe680 x1862022099124096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767974.968200:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099124096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767974.968205:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775767974.968207:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775767974.968209:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099124096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767975.863639:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767976.887690:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767977.911628:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767978.935600:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767979.959713:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767980.087599:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767980.087610:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767980.087615:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767980.087628:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5441 00000100:00080000:3.0:1775767980.087655:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767980.087663:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775767980.087737:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099124224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767980.087768:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767980.087772:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767980.087773:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767980.087777:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5441 00000100:00080000:3.0:1775767980.087780:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767980.087782:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767980.087792:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767980.087803:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099124352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767980.087812:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767980.087815:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767980.087817:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767980.087820:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5441 00000100:00080000:3.0:1775767980.087822:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767980.087827:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775767980.087830:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099124224:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775767980.087831:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767980.087843:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099124480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767980.087856:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767980.087859:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767980.087860:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767980.087863:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5441 00000100:00080000:3.0:1775767980.087864:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767980.087866:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767980.087870:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767980.087878:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099124608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767980.087884:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767980.087887:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775767980.087888:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767980.087890:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5441 00000100:00080000:3.0:1775767980.087892:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767980.087895:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775767980.087897:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767980.087904:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099124736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767980.087943:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099124352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767980.087992:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099124480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767980.088009:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099124608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767980.088025:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099124736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767980.983617:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767982.007609:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767983.031606:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775767983.159700:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775767983.159738:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775767983.159754:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767983.159797:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767983.159807:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767983.159818:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775767983.159829:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775767983.159854:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767980/real 1775767983] req@ffff8b23864f2d80 x1862022099124480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767983.159881:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099124480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767983.159892:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767983.159897:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767983.159901:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099124480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767983.159910:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767980/real 1775767983] req@ffff8b23864f2680 x1862022099124608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767983.159941:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099124608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767983.159951:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767983.159955:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767983.159959:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099124608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767983.159967:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767980/real 1775767983] req@ffff8b23864f0380 x1862022099124736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767983.159979:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099124736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767983.159988:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767983.159991:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767983.159995:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099124736:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767983.160002:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767980/real 1775767983] req@ffff8b23864f0700 x1862022099124224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767983.160016:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099124224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767983.160024:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767983.160028:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767983.160031:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099124224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767983.160041:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767980/real 1775767983] req@ffff8b23864f3100 x1862022099124352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767983.160053:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099124352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767983.160061:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767983.160064:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767983.160067:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099124352:192.168.202.128@tcp:38:kworker.0 00010000:00010000:3.0:1775767984.055673:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767985.079598:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767985.207665:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767985.207675:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767985.207680:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767985.207691:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5446 00000100:00080000:2.0:1775767985.207703:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767985.207710:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767985.207759:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099124864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767985.207801:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767985.207806:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767985.207808:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767985.207813:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5446 00000100:00080000:2.0:1775767985.207818:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767985.207822:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767985.207842:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767985.207861:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099124992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767985.207877:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767985.207884:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775767985.207885:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099124864:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767985.207886:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767985.207896:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5446 00000100:00080000:2.0:1775767985.207901:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767985.207921:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767985.207927:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767985.207943:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099125120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767985.207955:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767985.207960:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767985.207962:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767985.207967:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5446 00000100:00080000:2.0:1775767985.207971:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767985.207975:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767985.207980:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767985.207994:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099125248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767985.208005:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775767985.208006:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767985.208013:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767985.208015:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767985.208021:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5446 00000100:00080000:2.0:1775767985.208026:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767985.208031:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767985.208038:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767985.208053:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099125376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767985.208076:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767985/real 1775767985] req@ffff8b23b9ea7480 x1862022099124864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767985.208103:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099124864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767985.208117:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767985.208123:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767985.208127:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099124864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767985.208166:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099124992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767985.208194:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767985.208209:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767985/real 1775767985] req@ffff8b23b9ea6300 x1862022099124992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767985.208220:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099124992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767985.208229:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767985.208233:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767985.208237:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099124992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767985.208244:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099125120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767985.208267:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767985.208279:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767985/real 1775767985] req@ffff8b23b9ea7100 x1862022099125120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767985.208293:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099125120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767985.208303:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767985.208313:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767985.208319:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099125120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767985.208325:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099125248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767985.208350:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767985.208361:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767985/real 1775767985] req@ffff8b23b9ea4380 x1862022099125248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767985.208372:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099125248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767985.208379:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767985.208383:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767985.208386:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099125248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767985.208392:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099125376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767985.208415:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767985.208425:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767985/real 1775767985] req@ffff8b23b9ea4700 x1862022099125376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767985.208445:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099125376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767985.208527:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767985.208532:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767985.208536:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099125376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767986.103620:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767987.127630:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767988.151656:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767989.175660:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767990.199656:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767990.327560:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767990.327568:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767990.327572:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767990.327582:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5451 00000100:00080000:2.0:1775767990.327592:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767990.327598:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767990.327639:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022099125504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767990.327661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767990.327664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767990.327665:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767990.327668:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5451 00000100:00080000:2.0:1775767990.327671:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767990.327673:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767990.327680:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767990.327691:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099125632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767990.327706:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767990.327710:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775767990.327711:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099125504:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775767990.327711:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767990.327717:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5451 00000100:00080000:2.0:1775767990.327721:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767990.327724:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767990.327727:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767990.327735:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099125760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767990.327741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767990.327743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767990.327744:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767990.327747:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5451 00000100:00080000:2.0:1775767990.327750:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767990.327752:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767990.327755:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767990.327762:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099125888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767990.327767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767990.327770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767990.327771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767990.327773:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5451 00000100:00080000:2.0:1775767990.327777:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767990.327779:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767990.327782:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767990.327786:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775767990.327793:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099126016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767990.327825:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767990/real 1775767990] req@ffff8b23b9ea4e00 x1862022099125504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767990.327848:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022099125504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767990.327869:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767990.327873:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767990.327876:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099125504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767990.327894:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099125632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767990.327912:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767990.327921:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767990/real 1775767990] req@ffff8b23b9ea4700 x1862022099125632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767990.327929:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099125632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767990.327935:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767990.327937:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767990.327939:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099125632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767990.327946:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099125760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767990.327964:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767990.327974:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767990/real 1775767990] req@ffff8b23b9ea4380 x1862022099125760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767990.327987:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099125760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767990.327996:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767990.327999:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767990.328003:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099125760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767990.328009:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099125888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767990.328029:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767990.328037:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767990/real 1775767990] req@ffff8b23b9ea7100 x1862022099125888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767990.328045:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099125888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767990.328050:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767990.328052:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767990.328054:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099125888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767990.328058:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099126016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767990.328068:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767990.328073:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767990/real 1775767990] req@ffff8b23b9ea6300 x1862022099126016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767990.328081:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099126016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767990.328086:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767990.328087:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767990.328089:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099126016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775767991.223681:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767992.247632:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767993.271633:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767994.295619:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767995.319697:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767995.447532:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767995.447542:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767995.447547:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767995.447558:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5456 00000100:00080000:2.0:1775767995.447571:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767995.447577:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775767995.447622:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099126144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767995.447653:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767995.447658:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767995.447660:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767995.447665:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5456 00000100:00080000:2.0:1775767995.447669:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767995.447675:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767995.447701:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767995.447712:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099126144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775767995.447719:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099126272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767995.447734:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767995.447739:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767995.447741:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767995.447745:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5456 00000100:00080000:2.0:1775767995.447750:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767995.447755:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767995.447760:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767995.447775:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099126400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767995.447786:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767995.447790:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767995.447791:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767995.447797:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5456 00000100:00080000:2.0:1775767995.447801:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767995.447804:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767995.447826:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767995.447827:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775767995.447841:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099126528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767995.447849:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767995.447852:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775767995.447854:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767995.447856:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5456 00000100:00080000:2.0:1775767995.447859:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767995.447862:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775767995.447866:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767995.447874:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099126656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767995.447885:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767995/real 1775767995] req@ffff8b23b9ea7480 x1862022099126144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767995.447909:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099126144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767995.447922:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767995.447927:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767995.447932:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099126144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775767995.447974:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099126272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767995.447998:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767995.448008:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767995/real 1775767995] req@ffff8b23b9ea5880 x1862022099126272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767995.448017:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099126272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767995.448027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767995.448030:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767995.448032:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099126272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767995.448037:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099126400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775767995.448056:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767995.448067:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767995/real 1775767995] req@ffff8b23b9ea4000 x1862022099126400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767995.448080:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099126400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767995.448089:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767995.448093:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767995.448096:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099126400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775767995.448101:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099126528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767995.448120:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767995.448130:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767995/real 1775767995] req@ffff8b23b9ea6a00 x1862022099126528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767995.448141:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099126528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767995.448149:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767995.448153:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767995.448158:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099126528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775767995.448164:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099126656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775767995.448185:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775767995.448195:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767995/real 1775767995] req@ffff8b23b9ea4a80 x1862022099126656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767995.448204:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099126656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767995.448210:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767995.448212:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775767995.448214:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099126656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775767996.343685:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767997.367669:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767998.391659:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767999.415703:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768000.439664:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768000.567612:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768000.567622:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768000.567627:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768000.567637:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5461 00000100:00080000:3.0:1775768000.567651:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768000.567660:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768000.567713:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099126784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768000.567743:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768000.567746:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768000.567748:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768000.567751:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5461 00000100:00080000:3.0:1775768000.567754:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768000.567772:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768000.567782:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768000.567796:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099126912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768000.567806:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768000.567809:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768000.567810:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768000.567814:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5461 00000100:00080000:3.0:1775768000.567817:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768000.567830:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768000.567832:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099126784:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775768000.567838:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768000.567848:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099127040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768000.567856:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768000.567859:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768000.567861:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768000.567864:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5461 00000100:00080000:3.0:1775768000.567866:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768000.567868:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768000.567871:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768000.567880:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099127168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768000.567886:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768000.567888:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768000.567890:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768000.567892:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5461 00000100:00080000:3.0:1775768000.567895:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768000.567897:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768000.567900:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768000.567907:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099127296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768000.567919:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768000.567963:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768000/real 1775768000] req@ffff8b23864f1c00 x1862022099126784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768000.567980:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099126784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768000.567989:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768000.567993:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768000.567997:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099126784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768000.568002:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099126912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768000.568030:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768000.568039:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768000/real 1775768000] req@ffff8b23864f1f80 x1862022099126912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768000.568047:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099126912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768000.568052:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768000.568054:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768000.568057:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099126912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768000.568078:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099127040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768000.568090:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768000.568096:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768000/real 1775768000] req@ffff8b23864f0a80 x1862022099127040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768000.568103:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099127040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768000.568109:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768000.568111:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768000.568113:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099127040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768000.568116:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099127168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768000.568126:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768000.568132:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768000/real 1775768000] req@ffff8b23864f3100 x1862022099127168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768000.568143:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099127168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768000.568149:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768000.568150:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768000.568152:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099127168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768000.568155:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099127296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768000.568164:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768000.568169:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768000/real 1775768000] req@ffff8b23864f0700 x1862022099127296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768000.568175:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099127296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768000.568180:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768000.568182:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768000.568184:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099127296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768001.463655:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768002.487593:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768003.511656:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768004.535594:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768005.559646:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768005.687552:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768005.687563:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768005.687567:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768005.687577:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5467 00000100:00080000:2.0:1775768005.687590:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768005.687596:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768005.687638:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099127424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768005.687662:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768005.687665:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768005.687666:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768005.687669:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5467 00000100:00080000:2.0:1775768005.687671:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768005.687673:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768005.687682:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768005.687690:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099127552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768005.687698:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768005.687701:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768005.687702:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768005.687705:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5467 00000100:00080000:2.0:1775768005.687723:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768005.687728:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768005.687733:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099127424:192.168.202.128@tcp:250:kworker.0 00000008:00100000:2.0:1775768005.687735:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768005.687752:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099127680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768005.687763:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768005.687767:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768005.687769:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768005.687773:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5467 00000100:00080000:2.0:1775768005.687777:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768005.687781:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768005.687787:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768005.687796:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099127808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768005.687802:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768005.687805:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768005.687806:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768005.687809:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5467 00000100:00080000:2.0:1775768005.687811:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768005.687815:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775768005.687820:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:2.0:1775768005.687821:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768005.687829:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099127936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768005.687884:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768005/real 1775768005] req@ffff8b23b9ea7b80 x1862022099127424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768005.687903:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099127424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768005.687919:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768005.687923:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768005.687928:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099127424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768005.687936:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099127552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768005.687955:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768005.687966:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768005/real 1775768005] req@ffff8b23b9ea6680 x1862022099127552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768005.687977:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099127552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768005.687985:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768005.687989:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768005.687992:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099127552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768005.688020:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099127680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768005.688038:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768005.688047:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768005/real 1775768005] req@ffff8b23b9ea5f80 x1862022099127680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768005.688058:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099127680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768005.688066:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768005.688069:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768005.688072:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099127680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768005.688078:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099127808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768005.688093:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768005.688100:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768005/real 1775768005] req@ffff8b23b9ea6d80 x1862022099127808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768005.688111:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099127808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768005.688118:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768005.688122:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768005.688125:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099127808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768005.688130:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099127936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768005.688144:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768005.688151:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768005/real 1775768005] req@ffff8b23b9ea5c00 x1862022099127936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768005.688161:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099127936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768005.688169:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768005.688172:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768005.688175:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099127936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768006.583623:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768007.607722:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768008.631687:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768009.655615:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768010.679655:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768010.807555:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768010.807568:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768010.807573:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768010.807586:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5472 00000100:00080000:3.0:1775768010.807599:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768010.807607:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768010.807672:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099128064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768010.807706:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768010.807710:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768010.807712:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768010.807716:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5472 00000100:00080000:3.0:1775768010.807720:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768010.807724:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768010.807735:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768010.807748:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099128064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768010.807751:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099128192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768010.807766:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768010.807770:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768010.807772:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768010.807783:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5472 00000100:00080000:3.0:1775768010.807787:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768010.807792:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768010.807797:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768010.807809:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099128320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768010.807819:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775768010.807824:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775768010.807825:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768010.807830:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768010.807835:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5472 00000100:00080000:3.0:1775768010.807840:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768010.807844:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768010.807849:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768010.807861:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099128448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768010.807870:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768010.807874:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1775768010.807877:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768010/real 1775768010] req@ffff8b23864f0380 x1862022099128064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768010.807877:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768010.807884:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5472 00000100:00080000:3.0:1775768010.807888:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768010.807892:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768010.807899:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768010.807911:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099128576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768010.807917:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099128064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768010.807931:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768010.807936:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768010.807942:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099128064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768010.807970:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099128192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768010.807993:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768010.808005:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768010/real 1775768010] req@ffff8b23864f2680 x1862022099128192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768010.808016:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099128192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768010.808025:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768010.808028:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768010.808031:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099128192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768010.808038:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099128320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768010.808055:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768010.808064:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768010/real 1775768010] req@ffff8b23864f2d80 x1862022099128320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768010.808082:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099128320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768010.808090:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768010.808094:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768010.808097:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099128320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768010.808102:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099128448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768010.808121:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768010.808129:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768010/real 1775768010] req@ffff8b23864f1180 x1862022099128448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768010.808140:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099128448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768010.808148:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768010.808151:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768010.808154:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099128448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768010.808160:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099128576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768010.808175:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768010.808184:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768010/real 1775768010] req@ffff8b23864f3b80 x1862022099128576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768010.808195:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099128576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768010.808209:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768010.808212:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768010.808216:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099128576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768011.703607:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768012.727685:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768013.751686:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768014.775655:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768015.799658:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768015.927574:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768015.927584:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768015.927587:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768015.927597:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5477 00000100:00080000:3.0:1775768015.927623:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768015.927630:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768015.927674:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099128704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768015.927708:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768015.927712:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768015.927714:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768015.927719:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5477 00000100:00080000:3.0:1775768015.927723:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768015.927727:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768015.927738:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768015.927752:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099128832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768015.927756:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099128704:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768015.927772:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768015.927775:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768015.927776:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768015.927780:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5477 00000100:00080000:3.0:1775768015.927784:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768015.927787:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768015.927792:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768015.927804:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099128960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768015.927810:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768015.927813:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768015.927814:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768015.927817:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5477 00000100:00080000:3.0:1775768015.927820:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768015.927822:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768015.927825:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768015.927830:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768015.927834:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099129088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768015.927841:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768015.927843:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768015.927844:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768015.927847:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5477 00000100:00080000:3.0:1775768015.927849:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768015.927851:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768015.927854:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768015.927860:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099129216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768015.927882:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768015/real 1775768015] req@ffff8b23864f3480 x1862022099128704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768015.927901:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099128704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768015.927909:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768015.927913:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768015.927917:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099128704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768015.927951:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099128832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768015.927974:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768015.927983:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768015/real 1775768015] req@ffff8b23864f2a00 x1862022099128832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768015.927991:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099128832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768015.927996:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768015.927999:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768015.928001:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099128832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768015.928005:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099128960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768015.928015:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768015.928021:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768015/real 1775768015] req@ffff8b23864f0000 x1862022099128960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768015.928027:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099128960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768015.928032:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768015.928034:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768015.928035:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099128960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768015.928039:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099129088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768015.928050:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768015.928055:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768015/real 1775768015] req@ffff8b23864f3b80 x1862022099129088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768015.928061:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099129088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768015.928066:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768015.928067:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768015.928069:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099129088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768015.928072:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099129216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768015.928081:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768015.928086:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768015/real 1775768015] req@ffff8b23864f1180 x1862022099129216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768015.928092:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099129216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768015.928097:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768015.928098:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768015.928100:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099129216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768016.823740:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768017.847712:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768018.871766:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768019.895628:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768020.919671:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768021.047589:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768021.047600:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768021.047605:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768021.047616:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5482 00000100:00080000:3.0:1775768021.047631:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768021.047640:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768021.047689:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099129344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768021.047721:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768021.047724:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768021.047725:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768021.047728:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5482 00000100:00080000:3.0:1775768021.047731:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768021.047734:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768021.047744:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768021.047755:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099129472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768021.047764:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768021.047767:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768021.047769:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768021.047771:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5482 00000100:00080000:3.0:1775768021.047773:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768021.047778:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768021.047781:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768021.047788:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099129344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768021.047788:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099129600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768021.047801:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768021.047804:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768021.047806:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768021.047808:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5482 00000100:00080000:3.0:1775768021.047811:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768021.047813:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768021.047816:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768021.047824:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099129728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768021.047830:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768021.047832:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768021.047833:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768021.047835:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5482 00000100:00080000:3.0:1775768021.047840:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768021.047842:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768021.047845:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768021.047850:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099129856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768021.047867:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768021.047920:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768021/real 1775768021] req@ffff8b23864f2d80 x1862022099129344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768021.047942:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099129344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768021.047951:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768021.047954:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768021.047958:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099129344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768021.047964:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099129472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768021.047981:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768021.047990:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768021/real 1775768021] req@ffff8b23864f2680 x1862022099129472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768021.047999:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099129472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768021.048004:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768021.048006:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768021.048009:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099129472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768021.048032:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099129600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768021.048042:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768021.048048:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768021/real 1775768021] req@ffff8b23864f0380 x1862022099129600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768021.048056:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099129600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768021.048061:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768021.048063:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768021.048065:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099129600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768021.048069:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099129728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768021.048086:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768021.048096:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768021/real 1775768021] req@ffff8b23864f0700 x1862022099129728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768021.048109:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099129728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768021.048118:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768021.048122:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768021.048125:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099129728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768021.048132:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099129856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768021.048151:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768021.048159:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768021/real 1775768021] req@ffff8b23864f3100 x1862022099129856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768021.048167:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099129856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768021.048181:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768021.048183:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768021.048186:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099129856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768021.943627:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768022.967642:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768023.991610:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768025.015597:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768026.039653:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768026.167603:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768026.167614:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768026.167619:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768026.167630:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5487 00000100:00080000:0.0:1775768026.167644:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768026.167650:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768026.167696:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022099129984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768026.167731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768026.167734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768026.167735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768026.167738:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5487 00000100:00080000:0.0:1775768026.167741:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768026.167744:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768026.167755:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768026.167767:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022099130112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768026.167773:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099129984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775768026.167778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768026.167783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768026.167784:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768026.167788:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5487 00000100:00080000:0.0:1775768026.167791:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768026.167793:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768026.167797:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768026.167807:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022099130240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768026.167813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768026.167817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768026.167818:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768026.167821:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5487 00000100:00080000:0.0:1775768026.167824:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768026.167827:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768026.167830:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768026.167837:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022099130368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768026.167842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768026.167845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768026.167847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768026.167851:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5487 00000100:00080000:0.0:1775768026.167855:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775768026.167860:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775768026.167864:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768026.167871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768026.167886:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099130496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768026.167915:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768026/real 1775768026] req@ffff8b2382dafb80 x1862022099129984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768026.167937:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022099129984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768026.167950:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768026.167955:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768026.167960:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099129984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768026.167990:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099130112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768026.168012:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768026.168031:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768026/real 1775768026] req@ffff8b2382dadf80 x1862022099130112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768026.168044:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022099130112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768026.168053:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768026.168057:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768026.168061:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099130112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768026.168067:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099130240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768026.168086:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768026.168096:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768026/real 1775768026] req@ffff8b2382dace00 x1862022099130240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768026.168108:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022099130240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768026.168118:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768026.168122:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768026.168125:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099130240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768026.168131:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099130368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768026.168150:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768026.168159:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768026/real 1775768026] req@ffff8b2382dac700 x1862022099130368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768026.168172:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022099130368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768026.168181:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768026.168185:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768026.168189:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099130368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768026.168195:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099130496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768026.168212:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768026.168221:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768026/real 1775768026] req@ffff8b2382daf480 x1862022099130496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768026.168233:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099130496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768026.168243:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768026.168246:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768026.168250:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099130496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768027.063644:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768028.087698:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768029.111617:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768030.135662:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768031.159681:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768031.287589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768031.287600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768031.287605:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768031.287616:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5492 00000100:00080000:2.0:1775768031.287628:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768031.287635:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768031.287683:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022099130624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768031.287708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768031.287711:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768031.287713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768031.287716:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5492 00000100:00080000:2.0:1775768031.287718:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768031.287721:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768031.287731:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768031.287741:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099130752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768031.287751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768031.287754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768031.287755:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768031.287759:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5492 00000100:00080000:2.0:1775768031.287762:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768031.287764:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768031.287769:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768031.287775:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099130624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775768031.287779:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099130880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768031.287791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768031.287794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768031.287795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768031.287798:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5492 00000100:00080000:2.0:1775768031.287800:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768031.287802:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768031.287805:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768031.287812:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099131008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768031.287818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768031.287821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768031.287822:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768031.287825:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5492 00000100:00080000:2.0:1775768031.287827:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768031.287829:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768031.287832:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768031.287839:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099131136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775768031.287866:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768031.287928:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768031/real 1775768031] req@ffff8b23b9ea5500 x1862022099130624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768031.287951:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022099130624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768031.287969:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768031.287974:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768031.287979:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099130624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768031.287996:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099130752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768031.288024:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768031.288038:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768031/real 1775768031] req@ffff8b23b9ea5c00 x1862022099130752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768031.288050:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099130752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768031.288058:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768031.288061:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768031.288065:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099130752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768031.288092:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099130880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768031.288112:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768031.288122:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768031/real 1775768031] req@ffff8b23b9ea6d80 x1862022099130880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768031.288134:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099130880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768031.288142:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768031.288146:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768031.288149:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099130880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768031.288156:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099131008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768031.288173:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768031.288183:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768031/real 1775768031] req@ffff8b23b9ea5f80 x1862022099131008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768031.288196:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099131008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768031.288205:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768031.288208:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768031.288212:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099131008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768031.288218:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099131136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768031.288236:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768031.288245:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768031/real 1775768031] req@ffff8b23b9ea6680 x1862022099131136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768031.288258:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099131136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768031.288267:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768031.288270:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768031.288274:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099131136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768032.183693:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768033.207673:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768034.231687:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768035.255593:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768036.279620:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768036.407595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768036.407606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768036.407611:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768036.407621:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5497 00000100:00080000:0.0:1775768036.407633:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768036.407640:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768036.407685:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099131264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768036.407714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768036.407717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768036.407719:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768036.407726:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5497 00000100:00080000:0.0:1775768036.407729:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768036.407732:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768036.407741:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768036.407752:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099131392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768036.407762:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768036.407765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768036.407766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768036.407772:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5497 00000100:00100000:3.0:1775768036.407772:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099131264:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775768036.407777:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768036.407782:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768036.407785:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768036.407795:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099131520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768036.407802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768036.407805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768036.407806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768036.407810:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5497 00000100:00080000:0.0:1775768036.407813:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768036.407815:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768036.407819:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768036.407825:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099131648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768036.407830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768036.407833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768036.407834:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768036.407838:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5497 00000100:00080000:0.0:1775768036.407841:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768036.407843:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768036.407846:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775768036.407850:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768036.407859:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099131776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768036.407898:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768036/real 1775768036] req@ffff8b2382daed80 x1862022099131264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768036.407921:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099131264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768036.407934:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768036.407939:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768036.407945:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099131264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768036.407972:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099131392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768036.407991:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768036.408000:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768036/real 1775768036] req@ffff8b2382daea00 x1862022099131392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768036.408009:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099131392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768036.408015:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768036.408017:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768036.408020:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099131392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768036.408033:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099131520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768036.408046:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768036.408053:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768036/real 1775768036] req@ffff8b2382dad500 x1862022099131520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768036.408060:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099131520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768036.408070:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768036.408072:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768036.408075:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099131520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768036.408078:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099131648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768036.408089:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768036.408095:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768036/real 1775768036] req@ffff8b2382dae680 x1862022099131648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768036.408102:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099131648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768036.408107:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768036.408109:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768036.408111:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099131648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768036.408115:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099131776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768036.408125:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768036.408135:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768036/real 1775768036] req@ffff8b2382daca80 x1862022099131776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768036.408143:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099131776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768036.408149:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768036.408151:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768036.408153:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099131776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768037.303610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768038.327658:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768039.351624:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768040.375739:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768041.399592:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768041.527556:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768041.527568:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768041.527572:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768041.527582:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5502 00000100:00080000:3.0:1775768041.527595:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768041.527602:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768041.527648:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099131904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768041.527678:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768041.527682:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768041.527684:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768041.527688:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5502 00000100:00080000:3.0:1775768041.527692:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768041.527696:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768041.527708:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768041.527721:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099132032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768041.527726:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099131904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768041.527734:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768041.527737:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768041.527739:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768041.527743:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5502 00000100:00080000:3.0:1775768041.527747:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768041.527751:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768041.527756:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768041.527765:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099132160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768041.527773:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768041.527777:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768041.527778:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768041.527782:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5502 00000100:00080000:3.0:1775768041.527785:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768041.527788:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768041.527793:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768041.527802:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099132288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768041.527805:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775768041.527811:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768041.527814:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768041.527815:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768041.527819:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5502 00000100:00080000:3.0:1775768041.527822:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768041.527825:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768041.527829:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768041.527838:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099132416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768041.527854:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768041/real 1775768041] req@ffff8b23864f0a80 x1862022099131904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768041.527874:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099131904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768041.527886:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768041.527891:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768041.527895:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099131904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768041.527919:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099132032:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768041.527938:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768041.527948:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768041/real 1775768041] req@ffff8b23864f1f80 x1862022099132032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768041.527957:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099132032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768041.527963:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768041.527965:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768041.527967:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099132032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768041.527970:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099132160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768041.527982:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768041.527988:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768041/real 1775768041] req@ffff8b23864f1c00 x1862022099132160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768041.527995:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099132160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768041.528000:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768041.528002:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768041.528003:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099132160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768041.528008:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099132288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768041.528027:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768041.528038:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768041/real 1775768041] req@ffff8b23864f3100 x1862022099132288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768041.528050:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099132288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768041.528058:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768041.528062:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768041.528065:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099132288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768041.528072:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099132416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768041.528089:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768041.528096:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768041/real 1775768041] req@ffff8b23864f0700 x1862022099132416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768041.528103:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099132416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768041.528109:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768041.528111:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768041.528113:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099132416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768042.423680:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768043.447614:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768044.471649:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768045.495618:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768046.519611:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768046.647537:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768046.647549:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768046.647553:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768046.647564:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5508 00000100:00080000:3.0:1775768046.647576:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768046.647582:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768046.647629:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099132544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768046.647660:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768046.647664:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768046.647666:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768046.647670:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5508 00000100:00080000:3.0:1775768046.647673:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768046.647676:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768046.647687:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768046.647704:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099132544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768046.647705:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099132672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768046.647722:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768046.647724:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768046.647726:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768046.647728:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5508 00000100:00080000:3.0:1775768046.647730:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768046.647733:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768046.647735:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768046.647743:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099132800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768046.647751:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768046.647754:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768046.647758:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768046.647761:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5508 00000100:00080000:3.0:1775768046.647763:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768046.647766:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768046.647768:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768046.647774:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099132928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768046.647779:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768046.647782:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768046.647783:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768046.647785:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5508 00000100:00080000:3.0:1775768046.647787:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768046.647789:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768046.647791:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768046.647797:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099133056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768046.647854:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099132672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768046.647888:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099132800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768046.647910:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099132928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768046.647933:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099133056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768047.543687:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768048.567624:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768049.591683:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775768049.719784:0:2363:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775768049.719830:0:2363:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775768049.719848:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775768049.719896:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775768049.719907:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775768049.719916:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775768049.719925:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775768049.720019:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768046/real 1775768049] req@ffff8b23864f2680 x1862022099132672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768049.720044:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099132672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768049.720054:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768049.720059:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768049.720064:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099132672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768049.720071:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768046/real 1775768049] req@ffff8b23864f2d80 x1862022099132800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768049.720084:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099132800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768049.720093:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768049.720096:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768049.720100:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099132800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768049.720109:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768046/real 1775768049] req@ffff8b23864f1180 x1862022099132928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768049.720122:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099132928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768049.720131:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768049.720134:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768049.720138:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099132928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768049.720145:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768046/real 1775768049] req@ffff8b23864f3b80 x1862022099133056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768049.720157:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099133056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768049.720165:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768049.720168:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768049.720172:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099133056:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768049.720178:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768046/real 1775768049] req@ffff8b23864f0380 x1862022099132544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768049.720190:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099132544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768049.720197:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768049.720201:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768049.720204:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099132544:192.168.202.128@tcp:250:kworker.0 00010000:00010000:1.0:1775768050.615677:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768051.639652:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768051.767539:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768051.767549:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768051.767553:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768051.767563:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5513 00000100:00080000:2.0:1775768051.767575:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768051.767581:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768051.767618:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099133184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768051.767641:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768051.767644:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768051.767645:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768051.767648:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5513 00000100:00080000:2.0:1775768051.767650:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768051.767652:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768051.767661:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768051.767668:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099133312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768051.767676:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768051.767679:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768051.767680:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768051.767683:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5513 00000100:00080000:2.0:1775768051.767685:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768051.767688:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768051.767691:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768051.767698:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099133184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775768051.767698:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099133440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768051.767714:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768051.767716:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768051.767717:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768051.767718:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5513 00000100:00080000:2.0:1775768051.767720:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768051.767722:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768051.767725:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768051.767732:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099133568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768051.767737:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768051.767739:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768051.767740:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768051.767743:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5513 00000100:00080000:2.0:1775768051.767745:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768051.767748:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768051.767752:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768051.767763:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099133696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768051.767792:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768051.767846:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768051/real 1775768051] req@ffff8b23b9ea7b80 x1862022099133184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768051.767879:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099133184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768051.767891:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768051.767895:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768051.767900:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099133184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768051.767908:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099133312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768051.767933:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768051.767947:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768051/real 1775768051] req@ffff8b23b9ea4a80 x1862022099133312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768051.767957:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099133312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768051.767965:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768051.767968:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768051.767971:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099133312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768051.768001:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099133440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768051.768022:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768051.768033:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768051/real 1775768051] req@ffff8b23b9ea6a00 x1862022099133440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768051.768054:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099133440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768051.768062:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768051.768066:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768051.768070:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099133440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768051.768076:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099133568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768051.768098:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768051.768107:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768051/real 1775768051] req@ffff8b23b9ea4000 x1862022099133568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768051.768118:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099133568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768051.768126:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768051.768129:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768051.768133:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099133568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768051.768139:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099133696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768051.768157:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768051.768166:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768051/real 1775768051] req@ffff8b23b9ea5880 x1862022099133696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768051.768177:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099133696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768051.768185:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768051.768188:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768051.768191:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099133696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768052.663651:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768053.687676:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768054.711660:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768055.735696:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768056.759608:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768056.887591:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768056.887602:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768056.887607:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768056.887618:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5518 00000100:00080000:3.0:1775768056.887632:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768056.887640:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768056.887689:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099133824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768056.887724:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768056.887728:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768056.887730:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768056.887734:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5518 00000100:00080000:3.0:1775768056.887737:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768056.887741:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768056.887753:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768056.887768:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099133952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768056.887777:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099133824:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768056.887784:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768056.887788:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768056.887790:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768056.887795:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5518 00000100:00080000:3.0:1775768056.887798:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768056.887803:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768056.887808:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768056.887822:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099134080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768056.887831:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768056.887835:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768056.887837:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768056.887841:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5518 00000100:00080000:3.0:1775768056.887846:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775768056.887848:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775768056.887855:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768056.887861:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768056.887873:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099134208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768056.887883:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768056.887886:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768056.887888:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775768056.887892:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768056/real 1775768056] req@ffff8b23864f0000 x1862022099133824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768056.887895:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5518 00000100:00080000:3.0:1775768056.887902:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768056.887907:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768056.887913:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768056.887915:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099133824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768056.887927:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1775768056.887928:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099134336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768056.887935:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768056.887944:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099133824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768056.887977:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099133952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768056.887999:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768056.888011:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768056/real 1775768056] req@ffff8b23864f2a00 x1862022099133952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768056.888022:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099133952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768056.888029:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768056.888033:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768056.888036:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099133952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768056.888042:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099134080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768056.888061:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768056.888071:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768056/real 1775768056] req@ffff8b23864f3480 x1862022099134080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768056.888082:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099134080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768056.888090:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768056.888093:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768056.888106:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099134080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768056.888112:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099134208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768056.888133:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768056.888143:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768056/real 1775768056] req@ffff8b23864f0380 x1862022099134208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768056.888154:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099134208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768056.888162:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768056.888172:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768056.888175:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099134208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768056.888181:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099134336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768056.888202:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768056.888229:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768056/real 1775768056] req@ffff8b23864f3b80 x1862022099134336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768056.888241:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099134336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768056.888248:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768056.888251:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768056.888254:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099134336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768057.783682:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768058.807592:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768059.831643:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768060.855637:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768061.879619:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768062.007529:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768062.007540:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768062.007544:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768062.007553:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5523 00000100:00080000:3.0:1775768062.007566:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768062.007573:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768062.007621:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099134464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768062.007655:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768062.007658:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768062.007660:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768062.007664:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5523 00000100:00080000:3.0:1775768062.007668:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768062.007672:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768062.007685:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768062.007701:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099134592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768062.007702:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099134464:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768062.007713:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768062.007717:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768062.007718:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768062.007721:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5523 00000100:00080000:3.0:1775768062.007723:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768062.007726:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768062.007729:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768062.007737:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099134720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768062.007743:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768062.007746:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768062.007747:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768062.007750:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5523 00000100:00080000:3.0:1775768062.007754:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768062.007758:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768062.007763:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768062.007773:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099134848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768062.007782:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768062.007786:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768062.007787:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768062.007793:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5523 00000800:00000100:0.0:1775768062.007795:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775768062.007799:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768062.007804:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768062.007809:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768062.007818:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099134976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768062.007847:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768062/real 1775768062] req@ffff8b23864f1180 x1862022099134464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768062.007866:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099134464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768062.007877:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768062.007882:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768062.007886:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099134464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768062.007913:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099134592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768062.007936:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768062.007950:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768062/real 1775768062] req@ffff8b23864f2d80 x1862022099134592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768062.007962:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099134592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768062.007969:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768062.007972:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768062.007976:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099134592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768062.007981:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099134720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768062.008000:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768062.008009:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768062/real 1775768062] req@ffff8b23864f2680 x1862022099134720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768062.008020:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099134720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768062.008028:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768062.008031:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768062.008034:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099134720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768062.008040:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099134848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768062.008061:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768062.008070:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768062/real 1775768062] req@ffff8b23864f0700 x1862022099134848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768062.008081:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099134848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768062.008089:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768062.008092:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768062.008095:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099134848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768062.008100:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099134976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768062.008119:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768062.008128:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768062/real 1775768062] req@ffff8b23864f3100 x1862022099134976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768062.008138:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099134976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768062.008146:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768062.008149:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768062.008151:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099134976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768062.903609:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768063.927644:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768064.951676:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768065.975565:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768066.999643:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768067.127604:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768067.127615:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768067.127620:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768067.127631:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5528 00000100:00080000:2.0:1775768067.127645:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768067.127653:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768067.127715:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099135104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768067.127747:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768067.127751:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768067.127754:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768067.127757:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5528 00000100:00080000:2.0:1775768067.127761:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768067.127765:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768067.127776:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768067.127788:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099135232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768067.127793:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099135104:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775768067.127804:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768067.127807:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768067.127808:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768067.127811:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5528 00000100:00080000:2.0:1775768067.127814:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768067.127816:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768067.127819:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768067.127828:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099135360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768067.127836:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768067.127839:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768067.127841:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768067.127844:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5528 00000100:00080000:2.0:1775768067.127847:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768067.127849:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768067.127852:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768067.127859:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099135488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768067.127883:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768067.127885:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768067.127887:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775768067.127892:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775768067.127895:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5528 00000100:00080000:2.0:1775768067.127899:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768067.127902:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768067.127906:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768067.127915:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099135616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768067.127941:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768067/real 1775768067] req@ffff8b23b9ea7480 x1862022099135104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768067.127960:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099135104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768067.127969:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768067.127973:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768067.127976:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099135104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768067.128000:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099135232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768067.128033:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768067.128043:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768067/real 1775768067] req@ffff8b23b9ea6300 x1862022099135232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768067.128051:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099135232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768067.128057:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768067.128059:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768067.128062:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099135232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768067.128065:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099135360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768067.128077:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768067.128083:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768067/real 1775768067] req@ffff8b23b9ea7100 x1862022099135360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768067.128090:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099135360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768067.128096:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768067.128098:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768067.128099:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099135360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768067.128103:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099135488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768067.128137:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768067.128146:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768067/real 1775768067] req@ffff8b23b9ea4380 x1862022099135488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768067.128154:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099135488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768067.128160:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768067.128162:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768067.128165:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099135488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768067.128168:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099135616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768067.128182:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768067.128187:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768067/real 1775768067] req@ffff8b23b9ea4700 x1862022099135616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768067.128195:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099135616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768067.128200:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768067.128201:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768067.128203:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099135616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768068.023610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768069.047701:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768070.071708:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768071.095709:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768072.119690:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768072.247631:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768072.247639:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768072.247643:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768072.247653:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5533 00000100:00080000:3.0:1775768072.247663:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768072.247668:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768072.247700:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099135744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768072.247735:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768072.247742:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768072.247745:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768072.247749:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5533 00000100:00080000:3.0:1775768072.247753:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768072.247759:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768072.247770:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768072.247783:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099135872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768072.247792:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099135744:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768072.247796:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768072.247802:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768072.247804:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768072.247807:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5533 00000100:00080000:3.0:1775768072.247810:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768072.247813:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768072.247816:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768072.247837:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099136000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768072.247845:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768072.247847:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768072.247849:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768072.247851:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5533 00000100:00080000:3.0:1775768072.247854:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775768072.247856:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775768072.247859:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768072.247865:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768072.247875:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099136128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768072.247882:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768072.247884:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768072.247885:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768072.247888:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5533 00000100:00080000:3.0:1775768072.247891:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768072.247893:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768072.247896:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768072.247899:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768072/real 1775768072] req@ffff8b23864f1c00 x1862022099135744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768072.247907:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099136256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768072.247918:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099135744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768072.247925:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768072.247929:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768072.247933:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099135744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768072.247952:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099135872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768072.247968:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768072.247977:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768072/real 1775768072] req@ffff8b23864f1f80 x1862022099135872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768072.247984:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099135872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768072.247989:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768072.247991:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768072.247993:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099135872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768072.247997:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099136000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768072.248009:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768072.248014:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768072/real 1775768072] req@ffff8b23864f0a80 x1862022099136000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768072.248021:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099136000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768072.248031:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768072.248033:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768072.248035:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099136000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768072.248038:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099136128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768072.248049:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768072.248055:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768072/real 1775768072] req@ffff8b23864f3100 x1862022099136128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768072.248061:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099136128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768072.248091:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768072.248094:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768072.248096:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099136128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768072.248100:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099136256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768072.248115:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768072.248122:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768072/real 1775768072] req@ffff8b23864f0700 x1862022099136256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768072.248129:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099136256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768072.248134:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768072.248136:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768072.248138:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099136256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768073.143681:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768074.167688:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768075.191673:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768076.215667:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768077.239711:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768077.367571:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768077.367582:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768077.367588:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768077.367599:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5538 00000100:00080000:3.0:1775768077.367615:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768077.367623:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768077.367681:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099136384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768077.367715:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768077.367720:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768077.367722:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768077.367726:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5538 00000100:00080000:3.0:1775768077.367731:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768077.367736:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768077.367750:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768077.367770:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099136512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768077.367776:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099136384:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768077.367791:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768077.367795:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768077.367797:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768077.367803:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5538 00000100:00080000:3.0:1775768077.367809:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768077.367814:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768077.367820:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768077.367836:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099136640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768077.367845:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768077.367848:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768077.367850:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768077.367853:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5538 00000100:00080000:3.0:1775768077.367856:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768077.367859:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768077.367864:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768077.367866:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768077.367877:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099136768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768077.367895:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768077.367898:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768077.367900:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768077.367905:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5538 00000100:00080000:3.0:1775768077.367909:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768077.367914:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768077.367920:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768077.367935:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099136896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768077.367943:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768077/real 1775768077] req@ffff8b23864f2680 x1862022099136384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768077.367969:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099136384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768077.367982:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768077.367987:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768077.367992:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099136384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768077.368039:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099136512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768077.368065:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768077.368080:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768077/real 1775768077] req@ffff8b23864f2d80 x1862022099136512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768077.368094:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099136512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768077.368103:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768077.368107:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768077.368110:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099136512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768077.368117:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099136640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768077.368141:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768077.368152:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768077/real 1775768077] req@ffff8b23864f1180 x1862022099136640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768077.368164:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099136640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768077.368173:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768077.368176:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768077.368179:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099136640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768077.368185:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099136768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768077.368204:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768077.368213:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768077/real 1775768077] req@ffff8b23864f3b80 x1862022099136768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768077.368224:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099136768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768077.368233:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768077.368237:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768077.368240:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099136768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768077.368246:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099136896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768077.368265:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768077.368275:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768077/real 1775768077] req@ffff8b23864f0380 x1862022099136896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768077.368293:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099136896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768077.368301:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768077.368304:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768077.368308:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099136896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768078.263612:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768079.287698:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768080.311621:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768081.335667:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768082.359719:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768082.487589:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768082.487600:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768082.487605:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768082.487618:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5543 00000100:00080000:3.0:1775768082.487632:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768082.487640:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768082.487690:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099137024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768082.487724:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768082.487728:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768082.487730:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768082.487734:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5543 00000100:00080000:3.0:1775768082.487739:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768082.487743:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768082.487757:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768082.487776:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099137152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768082.487779:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099137024:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768082.487796:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768082.487801:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768082.487803:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768082.487808:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5543 00000100:00080000:3.0:1775768082.487812:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768082.487818:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768082.487824:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768082.487835:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099137280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768082.487842:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768082.487844:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768082.487846:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768082.487849:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5543 00000100:00080000:3.0:1775768082.487853:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768082.487855:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768082.487858:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768082.487864:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099137408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768082.487874:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768082.487879:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1775768082.487883:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775768082.487884:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768082.487892:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5543 00000100:00080000:3.0:1775768082.487897:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768082.487901:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768082.487908:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768082.487922:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099137536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768082.487941:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768082/real 1775768082] req@ffff8b23864f3480 x1862022099137024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768082.487981:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099137024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768082.487994:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768082.488000:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768082.488005:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099137024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768082.488035:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099137152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768082.488062:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768082.488075:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768082/real 1775768082] req@ffff8b23864f2a00 x1862022099137152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768082.488098:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099137152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768082.488107:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768082.488111:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768082.488115:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099137152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768082.488122:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099137280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768082.488152:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768082.488164:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768082/real 1775768082] req@ffff8b23864f0000 x1862022099137280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768082.488178:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099137280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768082.488187:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768082.488191:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768082.488195:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099137280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768082.488201:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099137408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768082.488224:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768082.488236:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768082/real 1775768082] req@ffff8b23864f0380 x1862022099137408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768082.488249:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099137408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768082.488265:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768082.488269:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768082.488272:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099137408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768082.488279:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099137536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768082.488298:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768082.488307:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768082/real 1775768082] req@ffff8b23864f3b80 x1862022099137536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768082.488318:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099137536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768082.488328:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768082.488331:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768082.488334:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099137536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768083.383713:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768084.407699:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768085.431678:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768086.455620:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768087.479632:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768087.607584:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768087.607594:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768087.607599:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768087.607610:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5549 00000100:00080000:0.0:1775768087.607622:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768087.607629:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768087.607666:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022099137664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768087.607696:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768087.607699:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768087.607701:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768087.607704:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5549 00000100:00080000:0.0:1775768087.607708:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768087.607712:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768087.607720:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768087.607725:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099137664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768087.607732:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099137792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768087.607742:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768087.607744:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768087.607746:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768087.607749:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5549 00000100:00080000:0.0:1775768087.607752:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768087.607754:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768087.607758:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768087.607767:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022099137920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768087.607773:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768087.607776:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768087.607777:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768087.607780:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5549 00000100:00080000:0.0:1775768087.607785:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775768087.607785:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775768087.607791:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768087.607795:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768087.607804:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022099138048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768087.607810:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768087.607812:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768087.607813:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768087.607816:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5549 00000100:00080000:0.0:1775768087.607818:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768087.607821:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768087.607824:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768087.607832:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099138176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768087.607837:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768087/real 1775768087] req@ffff8b2382dad880 x1862022099137664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768087.607861:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022099137664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768087.607871:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768087.607875:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768087.607878:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099137664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768087.607900:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099137792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768087.607937:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768087.607948:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768087/real 1775768087] req@ffff8b2382dac380 x1862022099137792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768087.607957:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099137792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768087.607967:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768087.607969:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768087.607972:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099137792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768087.607976:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099137920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768087.607994:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768087.608001:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768087/real 1775768087] req@ffff8b2382dac000 x1862022099137920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768087.608008:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022099137920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768087.608013:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768087.608016:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768087.608018:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099137920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768087.608021:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099138048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768087.608032:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768087.608038:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768087/real 1775768087] req@ffff8b2382daf100 x1862022099138048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768087.608047:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022099138048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768087.608056:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768087.608059:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768087.608062:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099138048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768087.608068:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099138176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768087.608089:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768087.608101:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768087/real 1775768087] req@ffff8b2382daca80 x1862022099138176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768087.608114:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099138176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768087.608123:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768087.608126:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768087.608130:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099138176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768088.503677:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768089.527609:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768090.551672:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768091.575644:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768092.599730:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768092.727625:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768092.727634:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768092.727638:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768092.727647:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5554 00000100:00080000:3.0:1775768092.727656:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768092.727661:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768092.727694:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099138304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768092.727728:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768092.727732:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768092.727734:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768092.727739:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5554 00000100:00080000:3.0:1775768092.727744:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768092.727749:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768092.727767:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768092.727782:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099138304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768092.727782:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099138432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768092.727814:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768092.727817:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768092.727818:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768092.727822:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5554 00000100:00080000:3.0:1775768092.727826:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768092.727832:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768092.727839:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768092.727856:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099138560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768092.727886:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768092.727890:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768092.727892:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768092.727896:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5554 00000100:00080000:3.0:1775768092.727901:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768092.727905:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768092.727912:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768092.727931:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768092.727933:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099138688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768092.727944:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768092.727947:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768092.727949:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768092.727955:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5554 00000100:00080000:3.0:1775768092.727962:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768092.727965:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768092.727972:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768092.727987:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099138816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768092.727996:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768092/real 1775768092] req@ffff8b23864f1180 x1862022099138304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768092.728041:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099138304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768092.728058:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768092.728063:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768092.728066:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099138304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768092.728095:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099138432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768092.728119:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768092.728133:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768092/real 1775768092] req@ffff8b23864f2d80 x1862022099138432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768092.728147:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099138432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768092.728156:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768092.728160:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768092.728163:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099138432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768092.728171:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099138560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768092.728191:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768092.728200:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768092/real 1775768092] req@ffff8b23864f2680 x1862022099138560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768092.728208:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099138560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768092.728214:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768092.728216:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768092.728218:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099138560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768092.728223:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099138688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768092.728243:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768092.728254:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768092/real 1775768092] req@ffff8b23864f0700 x1862022099138688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768092.728266:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099138688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768092.728274:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768092.728277:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768092.728280:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099138688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768092.728287:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099138816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768092.728309:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768092.728335:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768092/real 1775768092] req@ffff8b23864f3100 x1862022099138816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768092.728347:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099138816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768092.728356:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768092.728360:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768092.728363:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099138816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768093.623660:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768094.647626:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768095.671629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768096.695706:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768097.719663:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768097.847614:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768097.847625:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768097.847631:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768097.847643:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5559 00000100:00080000:1.0:1775768097.847657:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768097.847663:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768097.847707:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099138944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768097.847733:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768097.847736:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768097.847738:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768097.847741:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5559 00000100:00080000:1.0:1775768097.847743:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768097.847746:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768097.847755:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768097.847762:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099139072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768097.847771:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768097.847775:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768097.847776:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768097.847780:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5559 00000100:00080000:1.0:1775768097.847782:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768097.847784:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768097.847788:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768097.847800:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099139200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768097.847804:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099138944:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775768097.847807:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768097.847810:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768097.847811:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768097.847829:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5559 00000100:00080000:1.0:1775768097.847835:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768097.847838:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768097.847842:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768097.847855:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099139328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768097.847861:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768097.847863:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768097.847865:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768097.847867:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5559 00000100:00080000:1.0:1775768097.847870:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768097.847872:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768097.847874:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768097.847880:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099139456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768097.847958:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768097.848008:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768097/real 1775768097] req@ffff8b23864fdc00 x1862022099138944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768097.848029:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099138944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768097.848041:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768097.848046:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768097.848050:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099138944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768097.848060:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099139072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768097.848084:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768097.848097:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768097/real 1775768097] req@ffff8b23864fd500 x1862022099139072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768097.848106:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099139072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768097.848111:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768097.848114:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768097.848116:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099139072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768097.848143:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099139200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768097.848163:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768097.848173:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768097/real 1775768097] req@ffff8b23864ff100 x1862022099139200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768097.848192:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099139200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768097.848202:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768097.848205:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768097.848209:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099139200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768097.848215:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099139328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768097.848232:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768097.848240:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768097/real 1775768097] req@ffff8b23864fe300 x1862022099139328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768097.848248:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099139328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768097.848253:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768097.848256:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768097.848258:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099139328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768097.848261:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099139456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768097.848274:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768097.848283:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768097/real 1775768097] req@ffff8b23864ffb80 x1862022099139456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768097.848294:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099139456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768097.848310:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768097.848313:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768097.848317:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099139456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768098.743676:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768099.767663:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768100.791624:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768101.815689:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768102.839613:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768102.967579:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768102.967587:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768102.967590:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768102.967599:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5564 00000100:00080000:2.0:1775768102.967608:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768102.967613:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768102.967642:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022099139584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768102.967669:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768102.967672:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768102.967673:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768102.967675:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5564 00000100:00080000:2.0:1775768102.967677:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768102.967679:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768102.967690:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768102.967702:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099139712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768102.967710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768102.967712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768102.967713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768102.967716:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5564 00000100:00080000:2.0:1775768102.967719:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768102.967721:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768102.967725:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768102.967734:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099139584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775768102.967735:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099139840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768102.967756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768102.967758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768102.967758:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768102.967762:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5564 00000100:00080000:2.0:1775768102.967779:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768102.967784:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768102.967789:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768102.967799:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099139968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768102.967806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768102.967808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768102.967810:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768102.967812:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5564 00000100:00080000:2.0:1775768102.967815:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768102.967816:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768102.967825:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768102.967837:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775768102.967842:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099140096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768102.967894:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768102/real 1775768102] req@ffff8b23b9ea4e00 x1862022099139584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768102.967911:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022099139584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768102.967920:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768102.967923:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768102.967926:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099139584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768102.967932:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099139712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768102.967950:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768102.967959:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768102/real 1775768102] req@ffff8b23b9ea4700 x1862022099139712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768102.967967:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099139712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768102.967982:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768102.967986:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768102.967989:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099139712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768102.968026:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099139840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768102.968048:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768102.968059:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768102/real 1775768102] req@ffff8b23b9ea4380 x1862022099139840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768102.968071:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099139840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768102.968078:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768102.968091:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768102.968094:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099139840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768102.968100:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099139968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768102.968123:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768102.968134:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768102/real 1775768102] req@ffff8b23b9ea7100 x1862022099139968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768102.968146:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099139968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768102.968154:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768102.968157:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768102.968161:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099139968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768102.968169:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099140096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768102.968185:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768102.968193:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768102/real 1775768102] req@ffff8b23b9ea6300 x1862022099140096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768102.968201:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099140096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768102.968207:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768102.968209:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768102.968211:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099140096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768103.863622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768104.887616:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768105.911664:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768106.935615:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768107.959689:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768108.087566:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768108.087576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768108.087581:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768108.087593:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5569 00000100:00080000:1.0:1775768108.087607:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768108.087613:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768108.087660:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099140224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768108.087692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768108.087695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768108.087696:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768108.087700:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5569 00000100:00080000:1.0:1775768108.087702:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768108.087704:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768108.087713:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768108.087743:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099140352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768108.087752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768108.087755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768108.087756:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768108.087759:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5569 00000100:00080000:1.0:1775768108.087762:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768108.087766:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768108.087770:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768108.087778:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099140480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768108.087784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768108.087787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768108.087788:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768108.087791:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5569 00000100:00080000:1.0:1775768108.087793:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768108.087795:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768108.087798:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768108.087804:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099140608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768108.087811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768108.087814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768108.087815:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768108.087817:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5569 00000100:00080000:1.0:1775768108.087819:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768108.087825:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768108.087827:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099140224:192.168.202.128@tcp:250:kworker.0 00000008:00100000:1.0:1775768108.087828:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768108.087843:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099140736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768108.087914:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768108.087966:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768108/real 1775768108] req@ffff8b23864ffb80 x1862022099140224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768108.087987:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099140224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768108.087999:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768108.088004:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768108.088008:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099140224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768108.088016:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099140352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768108.088040:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768108.088054:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768108/real 1775768108] req@ffff8b23864fe300 x1862022099140352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768108.088063:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099140352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768108.088069:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768108.088071:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768108.088073:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099140352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768108.088078:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099140480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768108.088093:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768108.088099:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768108/real 1775768108] req@ffff8b23864ff100 x1862022099140480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768108.088107:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099140480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768108.088112:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768108.088114:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768108.088116:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099140480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768108.088120:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099140608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768108.088132:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768108.088137:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768108/real 1775768108] req@ffff8b23864fd500 x1862022099140608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768108.088150:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099140608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768108.088155:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768108.088157:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768108.088158:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099140608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768108.088187:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099140736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768108.088197:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768108.088203:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768108/real 1775768108] req@ffff8b23864fdc00 x1862022099140736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768108.088210:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099140736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768108.088215:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768108.088217:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768108.088219:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099140736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768108.983673:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768110.007725:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768111.031718:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768112.055743:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768113.079681:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768113.207626:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768113.207640:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768113.207645:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768113.207659:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5574 00000100:00080000:1.0:1775768113.207699:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768113.207707:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768113.207755:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099140864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768113.207783:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768113.207786:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768113.207788:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768113.207791:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5574 00000100:00080000:1.0:1775768113.207794:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768113.207796:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768113.207806:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768113.207815:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099140992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768113.207829:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768113.207833:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768113.207835:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768113.207840:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5574 00000100:00080000:1.0:1775768113.207844:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775768113.207850:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099140864:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775768113.207851:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768113.207861:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768113.207878:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099141120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768113.207887:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768113.207890:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768113.207891:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768113.207895:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5574 00000100:00080000:1.0:1775768113.207898:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768113.207900:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768113.207905:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768113.207913:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099141248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768113.207919:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768113.207922:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768113.207923:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768113.207926:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5574 00000100:00080000:1.0:1775768113.207928:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768113.207930:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768113.207933:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768113.207945:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099141376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768113.207994:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099140992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768113.208044:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099141120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768113.208077:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099141248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768113.208106:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099141376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768114.103730:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768115.127629:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768116.151681:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775768116.279759:0:2364:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775768116.279819:0:2364:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775768116.279842:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775768116.279895:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775768116.279906:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775768116.279917:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775768116.279928:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775768116.279963:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768113/real 1775768116] req@ffff8b23864fea00 x1862022099141120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768116.279994:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099141120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768116.280009:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768116.280016:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768116.280021:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099141120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768116.280030:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768113/real 1775768116] req@ffff8b23864fd180 x1862022099141248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768116.280043:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099141248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768116.280051:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768116.280055:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768116.280059:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099141248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768116.280066:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768113/real 1775768116] req@ffff8b23864fce00 x1862022099141376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768116.280079:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099141376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768116.280088:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768116.280092:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768116.280095:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099141376:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768116.280102:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768113/real 1775768116] req@ffff8b23864fe680 x1862022099140864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768116.280115:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099140864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768116.280124:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768116.280127:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768116.280131:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099140864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768116.280137:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768113/real 1775768116] req@ffff8b23864fca80 x1862022099140992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768116.280150:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099140992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768116.280157:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768116.280161:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768116.280165:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099140992:192.168.202.128@tcp:38:kworker.0 00010000:00010000:0.0:1775768117.175687:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768118.199669:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768118.327570:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768118.327581:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768118.327585:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768118.327594:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5579 00000100:00080000:3.0:1775768118.327606:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768118.327613:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768118.327663:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099141504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768118.327694:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768118.327698:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768118.327699:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768118.327702:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5579 00000100:00080000:3.0:1775768118.327705:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768118.327708:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768118.327719:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768118.327729:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099141632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768118.327746:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768118.327749:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099141504:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768118.327750:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768118.327754:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768118.327763:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5579 00000100:00080000:3.0:1775768118.327767:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768118.327771:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768118.327775:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768118.327793:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099141760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768118.327803:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768118.327807:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768118.327808:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768118.327812:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5579 00000100:00080000:3.0:1775768118.327816:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768118.327820:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768118.327826:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768118.327841:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768118.327841:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099141888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768118.327852:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768118.327855:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768118.327857:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768118.327860:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5579 00000100:00080000:3.0:1775768118.327865:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768118.327868:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768118.327873:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768118.327884:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099142016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768118.327893:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768118/real 1775768118] req@ffff8b23864f0a80 x1862022099141504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768118.327916:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099141504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768118.327927:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768118.327932:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768118.327936:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099141504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768118.327973:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099141632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768118.327996:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768118.328009:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768118/real 1775768118] req@ffff8b23864f1f80 x1862022099141632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768118.328021:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099141632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768118.328028:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768118.328031:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768118.328034:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099141632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768118.328038:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099141760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768118.328057:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768118.328066:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768118/real 1775768118] req@ffff8b23864f1c00 x1862022099141760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768118.328075:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099141760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768118.328088:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768118.328091:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768118.328094:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099141760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768118.328099:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099141888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768118.328119:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768118.328130:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768118/real 1775768118] req@ffff8b23864f3100 x1862022099141888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768118.328141:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099141888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768118.328149:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768118.328152:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768118.328155:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099141888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768118.328161:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099142016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768118.328179:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768118.328189:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768118/real 1775768118] req@ffff8b23864f0700 x1862022099142016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768118.328198:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099142016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768118.328205:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768118.328209:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768118.328212:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099142016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768119.223715:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768120.247633:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768121.271645:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768122.295641:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768123.319641:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768123.447598:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768123.447608:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768123.447612:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768123.447621:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5584 00000100:00080000:1.0:1775768123.447631:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768123.447637:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768123.447666:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099142144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768123.447686:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768123.447689:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768123.447690:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768123.447693:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5584 00000100:00080000:1.0:1775768123.447695:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768123.447699:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768123.447708:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768123.447722:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099142144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775768123.447723:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099142272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768123.447740:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768123.447744:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768123.447745:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768123.447750:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5584 00000100:00080000:1.0:1775768123.447754:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768123.447758:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768123.447764:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768123.447777:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775768123.447782:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099142400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768123.447793:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768123.447797:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768123.447799:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768123.447804:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5584 00000100:00100000:0.0:1775768123.447810:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768123/real 1775768123] req@ffff8b23864fdf80 x1862022099142144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768123.447810:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768123.447819:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768123.447825:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768123.447826:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099142144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768123.447838:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768123.447841:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768123.447843:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099142144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775768123.447852:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099142528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768123.447864:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099142272:192.168.202.128@tcp:38:kworker.0 00000100:00080000:1.0:1775768123.447865:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768123.447870:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768123.447872:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768123.447878:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5584 00000800:00000100:0.0:1775768123.447881:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775768123.447887:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775768123.447894:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768123/real 1775768123] req@ffff8b23864ff800 x1862022099142272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768123.447895:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768123.447907:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099142272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775768123.447907:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775768123.447916:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768123.447919:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768123.447921:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099142272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768123.447928:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099142400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:1.0:1775768123.447929:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099142656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768123.447943:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768123.447951:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768123/real 1775768123] req@ffff8b23864fed80 x1862022099142400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768123.447958:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099142400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768123.447963:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768123.447966:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768123.447968:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099142400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768123.447976:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099142528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768123.447986:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768123.447991:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768123/real 1775768123] req@ffff8b23864fc000 x1862022099142528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768123.447997:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099142528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768123.448001:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768123.448003:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768123.448005:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099142528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768123.448008:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099142656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768123.448025:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768123.448030:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768123/real 1775768123] req@ffff8b23864fc380 x1862022099142656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768123.448037:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099142656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768123.448041:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768123.448043:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768123.448045:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099142656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768124.343610:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768125.367647:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768126.391719:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768127.415710:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768128.439720:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768128.567574:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768128.567585:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768128.567589:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768128.567598:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5589 00000100:00080000:3.0:1775768128.567607:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768128.567614:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768128.567656:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099142784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768128.567682:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768128.567687:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768128.567688:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768128.567692:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5589 00000100:00080000:3.0:1775768128.567696:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768128.567700:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768128.567709:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768128.567720:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099142784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768128.567725:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099142912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768128.567738:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768128.567743:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768128.567746:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768128.567750:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5589 00000100:00080000:3.0:1775768128.567753:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768128.567757:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768128.567762:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768128.567773:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768128.567773:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099143040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768128.567781:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768128.567783:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768128.567785:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768128.567788:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5589 00000100:00080000:3.0:1775768128.567790:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768128.567792:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768128.567795:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768128.567802:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099143168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768128.567808:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768128/real 1775768128] req@ffff8b23864f2680 x1862022099142784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768128.567809:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768128.567815:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768128.567818:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768128.567824:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5589 00000100:00080000:3.0:1775768128.567827:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768128.567829:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768128.567833:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099142784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775768128.567833:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775768128.567844:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1775768128.567845:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099143296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768128.567849:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768128.567855:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099142784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768128.567874:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099142912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768128.567888:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768128.567895:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768128/real 1775768128] req@ffff8b23864f2d80 x1862022099142912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768128.567902:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099142912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768128.567907:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768128.567909:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768128.567911:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099142912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768128.567914:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099143040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768128.567923:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768128.567928:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768128/real 1775768128] req@ffff8b23864f1180 x1862022099143040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768128.567935:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099143040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768128.567939:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768128.567941:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768128.567943:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099143040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768128.567946:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099143168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768128.567955:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768128.567960:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768128/real 1775768128] req@ffff8b23864f3b80 x1862022099143168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768128.567974:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099143168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768128.567979:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768128.567981:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768128.567983:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099143168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768128.567987:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099143296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768128.567997:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768128.568002:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768128/real 1775768128] req@ffff8b23864f0380 x1862022099143296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768128.568008:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099143296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768128.568013:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768128.568014:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768128.568016:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099143296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768129.463589:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768130.487646:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768131.511646:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768132.535644:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768133.559646:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768133.687596:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768133.687609:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768133.687614:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768133.687625:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5595 00000100:00080000:3.0:1775768133.687639:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768133.687647:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768133.687697:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099143424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768133.687728:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768133.687732:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768133.687734:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768133.687738:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5595 00000100:00080000:3.0:1775768133.687741:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768133.687744:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768133.687753:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768133.687765:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099143552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768133.687773:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768133.687776:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768133.687777:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768133.687780:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5595 00000100:00080000:3.0:1775768133.687782:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775768133.687787:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099143424:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768133.687787:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768133.687793:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768133.687801:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099143680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768133.687807:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768133.687809:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768133.687811:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768133.687813:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5595 00000100:00080000:3.0:1775768133.687815:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768133.687817:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768133.687820:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768133.687827:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099143808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768133.687832:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768133.687835:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768133.687836:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768133.687838:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5595 00000100:00080000:3.0:1775768133.687840:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768133.687842:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768133.687844:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768133.687850:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099143936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768133.687868:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768133.687918:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768133/real 1775768133] req@ffff8b23864f0000 x1862022099143424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768133.687938:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099143424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768133.687950:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768133.687955:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768133.687959:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099143424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768133.687969:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099143552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768133.687994:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768133.688007:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768133/real 1775768133] req@ffff8b23864f2a00 x1862022099143552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768133.688020:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099143552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768133.688027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768133.688030:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768133.688034:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099143552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768133.688060:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099143680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768133.688087:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768133.688097:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768133/real 1775768133] req@ffff8b23864f3480 x1862022099143680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768133.688108:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099143680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768133.688116:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768133.688119:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768133.688122:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099143680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768133.688127:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099143808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768133.688148:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768133.688157:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768133/real 1775768133] req@ffff8b23864f0380 x1862022099143808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768133.688168:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099143808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768133.688176:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768133.688178:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768133.688181:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099143808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768133.688187:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099143936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768133.688207:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768133.688217:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768133/real 1775768133] req@ffff8b23864f3b80 x1862022099143936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768133.688229:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099143936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768133.688237:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768133.688240:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768133.688243:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099143936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768134.583662:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768135.607645:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768136.631600:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768137.655718:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768138.679602:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768138.807604:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768138.807618:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768138.807623:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768138.807637:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5600 00000100:00080000:2.0:1775768138.807649:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768138.807655:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768138.807695:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099144064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768138.807723:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768138.807726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768138.807728:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768138.807732:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5600 00000100:00080000:2.0:1775768138.807735:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768138.807739:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768138.807749:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768138.807762:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099144192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768138.807769:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099144064:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775768138.807782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768138.807786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768138.807788:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768138.807792:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5600 00000100:00080000:2.0:1775768138.807796:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768138.807799:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768138.807804:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768138.807813:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099144320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768138.807825:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768138.807828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768138.807829:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768138.807831:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5600 00000100:00080000:2.0:1775768138.807833:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775768138.807837:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775768138.807838:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768138.807843:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768138.807851:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099144448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768138.807857:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768138.807860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768138.807861:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768138.807863:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5600 00000100:00080000:2.0:1775768138.807864:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768138.807866:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768138.807869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768138.807879:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099144576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768138.807885:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768138/real 1775768138] req@ffff8b23b9ea7480 x1862022099144064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768138.807909:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099144064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768138.807918:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768138.807921:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768138.807924:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099144064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768138.807944:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099144192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768138.807958:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768138.807971:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768138/real 1775768138] req@ffff8b23b9ea5880 x1862022099144192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768138.807978:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099144192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768138.807983:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768138.807985:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768138.807987:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099144192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768138.807990:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099144320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768138.808001:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768138.808006:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768138/real 1775768138] req@ffff8b23b9ea4000 x1862022099144320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768138.808012:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099144320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768138.808017:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768138.808019:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768138.808020:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099144320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768138.808024:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099144448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768138.808035:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768138.808040:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768138/real 1775768138] req@ffff8b23b9ea6a00 x1862022099144448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768138.808046:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099144448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768138.808051:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768138.808052:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768138.808054:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099144448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768138.808057:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099144576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768138.808065:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768138.808070:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768138/real 1775768138] req@ffff8b23b9ea4a80 x1862022099144576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768138.808076:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099144576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768138.808080:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768138.808082:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768138.808084:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099144576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768139.703685:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768140.727653:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768141.751613:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768142.775781:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768143.799579:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768143.927604:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768143.927625:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768143.927629:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768143.927641:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5605 00000100:00080000:2.0:1775768143.927655:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768143.927662:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768143.927711:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099144704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768143.927745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768143.927749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768143.927751:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768143.927755:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5605 00000100:00080000:2.0:1775768143.927758:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768143.927762:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768143.927773:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768143.927787:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099144832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768143.927799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768143.927802:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099144704:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775768143.927804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768143.927806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768143.927815:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5605 00000100:00080000:2.0:1775768143.927819:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768143.927824:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768143.927829:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768143.927842:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099144960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768143.927851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768143.927856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768143.927858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768143.927862:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5605 00000100:00080000:2.0:1775768143.927866:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768143.927869:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768143.927875:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768143.927886:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099145088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768143.927895:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775768143.927899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768143.927903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768143.927905:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768143.927910:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5605 00000100:00080000:2.0:1775768143.927915:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768143.927920:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768143.927925:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768143.927937:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099145216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768143.927948:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768143/real 1775768143] req@ffff8b23b9ea7b80 x1862022099144704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768143.927970:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099144704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768143.927982:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768143.927987:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768143.927990:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099144704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768143.928014:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099144832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768143.928034:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768143.928045:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768143/real 1775768143] req@ffff8b23b9ea6680 x1862022099144832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768143.928057:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099144832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768143.928064:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768143.928067:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768143.928070:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099144832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768143.928076:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099144960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768143.928096:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768143.928105:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768143/real 1775768143] req@ffff8b23b9ea5f80 x1862022099144960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768143.928123:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099144960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768143.928131:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768143.928134:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768143.928137:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099144960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768143.928143:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099145088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768143.928162:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768143.928171:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768143/real 1775768143] req@ffff8b23b9ea6d80 x1862022099145088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768143.928182:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099145088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768143.928190:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768143.928193:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768143.928196:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099145088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768143.928200:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099145216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768143.928214:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768143.928221:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768143/real 1775768143] req@ffff8b23b9ea5c00 x1862022099145216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768143.928231:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099145216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768143.928244:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768143.928247:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768143.928250:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099145216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768144.823610:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768145.847684:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768146.871661:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768147.895679:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768148.919694:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768149.047611:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768149.047620:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768149.047625:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768149.047636:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5610 00000100:00080000:1.0:1775768149.047653:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768149.047660:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768149.047702:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099145344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768149.047730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768149.047734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768149.047735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768149.047739:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5610 00000100:00080000:1.0:1775768149.047743:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768149.047746:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768149.047755:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768149.047768:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099145472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768149.047781:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768149.047785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768149.047788:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775768149.047789:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099145344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775768149.047794:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5610 00000100:00080000:1.0:1775768149.047801:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768149.047805:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768149.047811:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768149.047824:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099145600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768149.047834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768149.047837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768149.047838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768149.047842:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5610 00000100:00080000:1.0:1775768149.047846:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768149.047853:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775768149.047854:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:1.0:1775768149.047860:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768149.047873:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099145728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768149.047882:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768149.047885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768149.047886:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768149.047890:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5610 00000100:00080000:1.0:1775768149.047894:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768149.047897:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768149.047903:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768149.047908:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768149/real 1775768149] req@ffff8b23864fc380 x1862022099145344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768149.047917:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099145856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768149.047929:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099145344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768149.047939:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768149.047943:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768149.047947:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099145344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768149.047969:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099145472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768149.047990:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768149.048001:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768149/real 1775768149] req@ffff8b23864fc000 x1862022099145472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768149.048011:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099145472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768149.048017:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768149.048021:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768149.048024:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099145472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768149.048029:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099145600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768149.048046:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768149.048055:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768149/real 1775768149] req@ffff8b23864fed80 x1862022099145600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768149.048065:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099145600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768149.048072:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768149.048075:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768149.048078:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099145600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768149.048084:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099145728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768149.048102:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768149.048110:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768149/real 1775768149] req@ffff8b23864ff800 x1862022099145728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768149.048121:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099145728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768149.048128:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768149.048131:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768149.048134:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099145728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768149.048139:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099145856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768149.048163:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768149.048172:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768149/real 1775768149] req@ffff8b23864fdf80 x1862022099145856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768149.048182:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099145856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768149.048189:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768149.048192:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768149.048195:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099145856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768149.943661:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768150.967646:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768151.991672:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768153.015665:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768154.039599:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768154.167606:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768154.167615:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768154.167619:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768154.167630:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5615 00000100:00080000:3.0:1775768154.167641:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768154.167647:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768154.167699:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099145984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768154.167732:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768154.167736:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768154.167738:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768154.167742:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5615 00000100:00080000:3.0:1775768154.167746:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768154.167749:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768154.167760:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768154.167772:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099146112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768154.167779:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099145984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768154.167793:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768154.167800:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768154.167802:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768154.167807:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5615 00000100:00080000:3.0:1775768154.167811:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768154.167815:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768154.167822:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768154.167834:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099146240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768154.167843:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768154.167845:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768154.167847:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768154.167850:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5615 00000100:00080000:3.0:1775768154.167853:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768154.167856:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768154.167859:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768154.167864:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768154.167871:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099146368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768154.167880:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768154.167883:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768154.167885:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768154.167889:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5615 00000100:00080000:3.0:1775768154.167893:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768154.167897:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768154.167903:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768154.167910:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768154/real 1775768154] req@ffff8b23864f1180 x1862022099145984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768154.167916:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099146496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768154.167933:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099145984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768154.167943:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768154.167947:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768154.167952:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099145984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768154.167979:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099146112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768154.168001:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768154.168012:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768154/real 1775768154] req@ffff8b23864f2d80 x1862022099146112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768154.168020:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099146112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768154.168026:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768154.168028:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768154.168040:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099146112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768154.168049:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099146240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768154.168069:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768154.168080:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768154/real 1775768154] req@ffff8b23864f2680 x1862022099146240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768154.168091:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099146240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768154.168099:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768154.168102:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768154.168105:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099146240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768154.168110:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099146368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768154.168128:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768154.168137:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768154/real 1775768154] req@ffff8b23864f0700 x1862022099146368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768154.168146:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099146368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768154.168154:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768154.168157:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768154.168160:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099146368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768154.168165:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099146496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768154.168183:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768154.168191:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768154/real 1775768154] req@ffff8b23864f3100 x1862022099146496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768154.168198:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099146496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768154.168203:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768154.168205:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768154.168207:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099146496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768155.063693:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768156.087601:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768157.111657:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768158.135734:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768159.159606:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768159.287529:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768159.287536:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768159.287540:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768159.287548:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5620 00000100:00080000:1.0:1775768159.287557:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768159.287563:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768159.287593:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099146624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768159.287613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768159.287615:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768159.287617:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768159.287619:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5620 00000100:00080000:1.0:1775768159.287621:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768159.287623:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768159.287631:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768159.287639:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099146752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768159.287646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768159.287648:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768159.287649:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768159.287652:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5620 00000100:00080000:1.0:1775768159.287654:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768159.287658:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768159.287660:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099146624:192.168.202.128@tcp:250:kworker.0 00000008:00100000:1.0:1775768159.287661:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768159.287672:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099146880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768159.287679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768159.287681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768159.287682:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768159.287685:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5620 00000100:00080000:1.0:1775768159.287686:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768159.287688:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768159.287691:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768159.287697:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099147008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768159.287704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768159.287706:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768159.287707:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768159.287710:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5620 00000800:00000100:0.0:1775768159.287713:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775768159.287714:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768159.287719:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768159.287721:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768159.287731:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099147136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768159.287758:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768159/real 1775768159] req@ffff8b23864fca80 x1862022099146624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768159.287777:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099146624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768159.287787:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768159.287792:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768159.287796:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099146624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768159.287803:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099146752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768159.287825:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768159.287834:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768159/real 1775768159] req@ffff8b23864fe680 x1862022099146752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768159.287843:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099146752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768159.287848:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768159.287850:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768159.287852:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099146752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768159.287870:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099146880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768159.287881:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768159.287887:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768159/real 1775768159] req@ffff8b23864fce00 x1862022099146880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768159.287900:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099146880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768159.287905:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768159.287907:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768159.287909:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099146880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768159.287913:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099147008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768159.287924:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768159.287929:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768159/real 1775768159] req@ffff8b23864fd180 x1862022099147008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768159.287936:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099147008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768159.287946:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768159.287948:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768159.287950:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099147008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768159.287953:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099147136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768159.287963:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768159.287968:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768159/real 1775768159] req@ffff8b23864fea00 x1862022099147136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768159.287974:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099147136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768159.287979:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768159.287980:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768159.287982:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099147136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768160.183705:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768161.207596:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768162.231719:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768163.255670:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768164.279716:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768164.407574:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768164.407584:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768164.407589:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768164.407600:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5625 00000100:00080000:2.0:1775768164.407612:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768164.407619:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768164.407670:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022099147264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768164.407704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768164.407709:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768164.407711:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768164.407715:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5625 00000100:00080000:2.0:1775768164.407720:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768164.407724:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768164.407734:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768164.407748:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099147392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768164.407758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768164.407763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775768164.407764:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099147264:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775768164.407765:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768164.407771:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5625 00000100:00080000:2.0:1775768164.407777:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768164.407780:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768164.407783:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768164.407790:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099147520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768164.407797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768164.407800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768164.407801:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768164.407804:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5625 00000100:00080000:2.0:1775768164.407807:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768164.407810:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768164.407813:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768164.407822:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099147648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768164.407828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768164.407831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768164.407832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768164.407835:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5625 00000100:00080000:2.0:1775768164.407837:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768164.407840:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768164.407842:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768164.407849:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099147776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768164.407864:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768164.407917:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768164/real 1775768164] req@ffff8b23b9ea5500 x1862022099147264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768164.407940:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022099147264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768164.407952:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768164.407957:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768164.407962:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099147264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768164.407992:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099147392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768164.408017:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768164.408031:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768164/real 1775768164] req@ffff8b23b9ea5c00 x1862022099147392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768164.408043:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099147392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768164.408050:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768164.408053:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768164.408057:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099147392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768164.408062:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099147520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768164.408083:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768164.408093:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768164/real 1775768164] req@ffff8b23b9ea6d80 x1862022099147520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768164.408106:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099147520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768164.408121:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768164.408125:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768164.408128:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099147520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768164.408135:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099147648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768164.408158:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768164.408170:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768164/real 1775768164] req@ffff8b23b9ea5f80 x1862022099147648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768164.408217:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099147648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768164.408226:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768164.408229:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768164.408233:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099147648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768164.408239:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099147776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768164.408262:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768164.408274:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768164/real 1775768164] req@ffff8b23b9ea6680 x1862022099147776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768164.408287:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099147776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768164.408295:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768164.408299:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768164.408303:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099147776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768165.303699:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768166.327700:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768167.351710:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768168.375673:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768169.399657:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768169.527562:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768169.527577:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768169.527582:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768169.527591:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5630 00000100:00080000:1.0:1775768169.527603:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768169.527608:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768169.527648:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099147904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768169.527674:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768169.527677:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768169.527680:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768169.527683:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5630 00000100:00080000:1.0:1775768169.527685:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768169.527688:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768169.527697:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768169.527707:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099148032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768169.527714:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768169.527717:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768169.527719:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768169.527722:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5630 00000100:00080000:1.0:1775768169.527724:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768169.527728:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768169.527729:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099147904:192.168.202.128@tcp:250:kworker.0 00000008:00100000:1.0:1775768169.527731:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768169.527740:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099148160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768169.527747:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768169.527750:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768169.527751:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768169.527754:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5630 00000100:00080000:1.0:1775768169.527755:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768169.527757:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768169.527760:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768169.527770:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099148288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768169.527776:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768169.527779:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768169.527780:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768169.527783:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5630 00000100:00080000:1.0:1775768169.527796:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768169.527802:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768169.527805:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768169.527811:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775768169.527817:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099148416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768169.527858:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768169/real 1775768169] req@ffff8b23864fdc00 x1862022099147904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768169.527878:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099147904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768169.527890:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768169.527894:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768169.527898:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099147904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768169.527906:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099148032:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768169.527927:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768169.527939:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768169/real 1775768169] req@ffff8b23864fd500 x1862022099148032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768169.527950:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099148032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768169.527958:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768169.527961:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768169.527964:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099148032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768169.527990:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099148160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768169.528008:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768169.528019:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768169/real 1775768169] req@ffff8b23864ff100 x1862022099148160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768169.528031:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099148160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768169.528039:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768169.528042:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768169.528045:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099148160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768169.528051:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099148288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768169.528068:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768169.528075:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768169/real 1775768169] req@ffff8b23864fe300 x1862022099148288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768169.528082:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099148288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768169.528087:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768169.528089:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768169.528091:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099148288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768169.528094:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099148416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768169.528103:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768169.528108:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768169/real 1775768169] req@ffff8b23864ffb80 x1862022099148416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768169.528118:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099148416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768169.528139:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768169.528141:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768169.528143:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099148416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768170.423704:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768171.447600:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768172.471655:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768173.495608:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768174.519669:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768174.647535:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768174.647544:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768174.647549:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768174.647559:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5636 00000100:00080000:1.0:1775768174.647571:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768174.647576:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768174.647618:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099148544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768174.647641:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768174.647644:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768174.647645:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768174.647648:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5636 00000100:00080000:1.0:1775768174.647650:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768174.647652:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768174.647662:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768174.647672:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099148672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768174.647681:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768174.647684:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775768174.647685:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099148544:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775768174.647686:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768174.647692:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5636 00000100:00080000:1.0:1775768174.647694:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768174.647696:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768174.647699:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768174.647709:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099148800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768174.647714:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768174.647717:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768174.647718:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768174.647721:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5636 00000100:00080000:1.0:1775768174.647742:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768174.647745:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768174.647754:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768174.647755:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775768174.647764:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099148928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768174.647771:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768174.647773:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768174.647776:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768174.647780:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5636 00000100:00080000:1.0:1775768174.647783:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768174.647787:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768174.647792:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768174.647805:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768174/real 1775768174] req@ffff8b23864ffb80 x1862022099148544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768174.647806:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099149056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768174.647836:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099148544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768174.647845:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768174.647849:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768174.647852:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099148544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768174.647878:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099148672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768174.647893:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768174.647901:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768174/real 1775768174] req@ffff8b23864fe300 x1862022099148672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768174.647913:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099148672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768174.647917:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768174.647920:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768174.647921:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099148672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768174.647925:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099148800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768174.647934:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768174.647939:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768174/real 1775768174] req@ffff8b23864ff100 x1862022099148800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768174.647946:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099148800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768174.647950:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768174.647952:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768174.647953:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099148800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768174.647956:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099148928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768174.647966:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768174.647971:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768174/real 1775768174] req@ffff8b23864fd500 x1862022099148928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768174.647977:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099148928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768174.647986:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768174.647987:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768174.647989:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099148928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768174.647992:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099149056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768174.648001:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768174.648005:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768174/real 1775768174] req@ffff8b23864fdc00 x1862022099149056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768174.648011:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099149056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768174.648015:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768174.648017:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768174.648019:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099149056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768175.543603:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768176.567652:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768177.591604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768178.615607:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768179.639710:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768179.767595:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768179.767609:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768179.767615:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768179.767628:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5641 00000100:00080000:3.0:1775768179.767645:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768179.767654:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768179.767707:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099149184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768179.767742:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768179.767745:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768179.767747:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768179.767750:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5641 00000100:00080000:3.0:1775768179.767753:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768179.767757:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768179.767772:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768179.767785:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099149312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768179.767795:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768179.767798:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099149184:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768179.767799:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768179.767802:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768179.767810:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5641 00000100:00080000:3.0:1775768179.767813:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768179.767815:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768179.767819:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768179.767830:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099149440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768179.767841:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768179.767845:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768179.767846:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768179.767849:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5641 00000100:00080000:3.0:1775768179.767852:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768179.767854:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768179.767856:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768179.767863:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099149568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768179.767869:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768179.767873:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768179.767874:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768179.767877:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5641 00000100:00080000:3.0:1775768179.767879:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768179.767881:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768179.767884:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768179.767893:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099149696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768179.767925:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099149312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768179.767949:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099149440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768179.767966:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099149568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768179.767984:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099149696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768180.663628:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768181.687712:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768182.711637:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775768182.839743:0:2365:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775768182.839790:0:2365:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775768182.839809:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775768182.839843:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775768182.839849:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775768182.839856:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775768182.839863:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775768182.839917:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768179/real 1775768182] req@ffff8b23864f1f80 x1862022099149312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768182.839944:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099149312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768182.839956:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768182.839962:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768182.839966:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099149312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768182.839974:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768179/real 1775768182] req@ffff8b23864f0a80 x1862022099149440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768182.839986:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099149440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768182.840028:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768182.840043:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768182.840047:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099149440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768182.840054:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768179/real 1775768182] req@ffff8b23864f3100 x1862022099149568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768182.840067:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099149568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768182.840076:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768182.840080:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768182.840084:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099149568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768182.840091:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768179/real 1775768182] req@ffff8b23864f0700 x1862022099149696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768182.840102:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099149696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768182.840110:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768182.840114:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768182.840117:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099149696:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768182.840124:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768179/real 1775768182] req@ffff8b23864f1c00 x1862022099149184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768182.840136:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099149184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768182.840144:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768182.840147:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768182.840150:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099149184:192.168.202.128@tcp:250:kworker.0 00010000:00010000:1.0:1775768183.735682:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768184.759737:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768184.887608:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768184.887631:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768184.887641:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768184.887654:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5646 00000100:00080000:3.0:1775768184.887672:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768184.887680:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768184.887732:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099149824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768184.887781:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768184.887785:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768184.887787:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768184.887793:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5646 00000100:00080000:3.0:1775768184.887798:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768184.887802:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768184.887814:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768184.887831:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099149952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768184.887843:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768184.887847:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768184.887849:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768184.887854:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5646 00000100:00080000:3.0:1775768184.887858:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768184.887863:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768184.887868:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768184.887882:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099150080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768184.887894:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768184.887898:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768184.887900:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768184.887905:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5646 00000100:00080000:3.0:1775768184.887910:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768184.887915:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768184.887917:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099149824:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775768184.887921:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768184.887937:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099150208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768184.887950:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768184.887955:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768184.887957:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768184.887962:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5646 00000100:00080000:3.0:1775768184.887967:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768184.887971:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768184.887993:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768184.888008:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099150336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768184.888122:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768184.888181:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768184/real 1775768184] req@ffff8b23864f2680 x1862022099149824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768184.888203:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099149824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768184.888217:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768184.888221:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768184.888227:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099149824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768184.888235:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099149952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768184.888263:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768184.888277:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768184/real 1775768184] req@ffff8b23864f2d80 x1862022099149952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768184.888289:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099149952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768184.888297:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768184.888300:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768184.888304:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099149952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768184.888310:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099150080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768184.888334:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768184.888345:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768184/real 1775768184] req@ffff8b23864f1180 x1862022099150080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768184.888355:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099150080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768184.888364:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768184.888368:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768184.888372:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099150080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768184.888413:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099150208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768184.888438:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768184.888451:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768184/real 1775768184] req@ffff8b23864f3b80 x1862022099150208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768184.888506:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099150208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768184.888515:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768184.888519:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768184.888524:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099150208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768184.888531:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099150336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768184.888556:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768184.888568:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768184/real 1775768184] req@ffff8b23864f0380 x1862022099150336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768184.888581:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099150336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768184.888600:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768184.888604:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768184.888608:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099150336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768185.783662:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768186.807722:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768187.831650:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768188.855646:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768189.879637:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768190.007567:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768190.007575:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768190.007580:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768190.007590:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5651 00000100:00080000:2.0:1775768190.007600:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768190.007606:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768190.007638:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099150464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768190.007664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768190.007670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768190.007672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768190.007676:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5651 00000100:00080000:2.0:1775768190.007680:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768190.007685:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768190.007695:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768190.007706:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099150464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775768190.007708:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099150592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768190.007725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768190.007728:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768190.007730:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768190.007734:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5651 00000100:00080000:2.0:1775768190.007738:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768190.007742:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768190.007747:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768190.007758:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099150720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768190.007769:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775768190.007771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768190.007781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768190.007783:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768190.007787:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5651 00000100:00080000:2.0:1775768190.007791:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768190.007795:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768190.007801:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768190.007810:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768190/real 1775768190] req@ffff8b23b9ea7b80 x1862022099150464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768190.007815:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099150848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768190.007835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768190.007839:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099150464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768190.007840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768190.007843:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768190.007851:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5651 00000100:00080000:0.0:1775768190.007856:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768190.007856:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768190.007862:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00080000:2.0:1775768190.007864:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768190.007868:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099150464:192.168.202.128@tcp:250:kworker.0 00000008:00100000:2.0:1775768190.007871:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768190.007885:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099150976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768190.007889:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099150592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768190.007913:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768190.007937:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768190/real 1775768190] req@ffff8b23b9ea4a80 x1862022099150592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768190.007947:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099150592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768190.007952:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768190.007954:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768190.007955:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099150592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768190.007959:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099150720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768190.007972:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768190.007978:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768190/real 1775768190] req@ffff8b23b9ea6a00 x1862022099150720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768190.007988:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099150720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768190.007992:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768190.007994:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768190.007996:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099150720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768190.007999:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099150848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768190.008008:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768190.008013:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768190/real 1775768190] req@ffff8b23b9ea4000 x1862022099150848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768190.008019:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099150848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768190.008023:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768190.008025:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768190.008026:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099150848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768190.008035:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099150976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768190.008044:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768190.008048:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768190/real 1775768190] req@ffff8b23b9ea5880 x1862022099150976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768190.008054:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099150976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768190.008062:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768190.008064:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768190.008065:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099150976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768190.903599:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768191.927631:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768192.951627:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768193.975660:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768194.999614:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768195.127569:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768195.127579:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768195.127583:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768195.127596:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5656 00000100:00080000:2.0:1775768195.127610:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768195.127619:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768195.127675:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099151104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768195.127705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768195.127708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768195.127710:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768195.127713:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5656 00000100:00080000:2.0:1775768195.127717:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768195.127721:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768195.127732:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768195.127747:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099151104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775768195.127754:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099151232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768195.127766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768195.127769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768195.127771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768195.127778:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5656 00000100:00080000:2.0:1775768195.127782:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768195.127785:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768195.127790:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768195.127803:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099151360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768195.127812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768195.127815:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768195.127817:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768195.127821:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5656 00000100:00080000:2.0:1775768195.127825:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768195.127829:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768195.127834:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768195.127839:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775768195.127845:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099151488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768195.127856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768195.127858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768195.127860:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768195.127863:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5656 00000100:00080000:2.0:1775768195.127865:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768195.127867:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768195.127870:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768195.127897:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099151616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768195.127903:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768195/real 1775768195] req@ffff8b23b9ea7480 x1862022099151104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768195.127926:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099151104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768195.127938:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768195.127942:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768195.127947:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099151104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768195.127973:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099151232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768195.127995:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768195.128009:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768195/real 1775768195] req@ffff8b23b9ea6300 x1862022099151232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768195.128021:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099151232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768195.128035:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768195.128038:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768195.128042:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099151232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768195.128048:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099151360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768195.128067:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768195.128077:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768195/real 1775768195] req@ffff8b23b9ea7100 x1862022099151360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768195.128089:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099151360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768195.128096:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768195.128100:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768195.128103:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099151360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768195.128109:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099151488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768195.128128:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768195.128138:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768195/real 1775768195] req@ffff8b23b9ea4380 x1862022099151488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768195.128149:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099151488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768195.128156:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768195.128160:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768195.128163:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099151488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768195.128169:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099151616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768195.128189:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768195.128199:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768195/real 1775768195] req@ffff8b23b9ea4700 x1862022099151616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768195.128211:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099151616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768195.128219:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768195.128222:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768195.128226:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099151616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768196.023632:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768197.047679:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768198.071664:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768199.095674:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768200.119603:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768200.247607:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768200.247618:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768200.247623:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768200.247635:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5661 00000100:00080000:2.0:1775768200.247649:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768200.247658:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768200.247708:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022099151744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768200.247745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768200.247750:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768200.247752:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768200.247756:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5661 00000100:00080000:2.0:1775768200.247761:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768200.247764:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768200.247776:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768200.247791:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099151872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768200.247800:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099151744:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775768200.247809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768200.247813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768200.247815:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768200.247821:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5661 00000100:00080000:2.0:1775768200.247842:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768200.247848:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768200.247854:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768200.247870:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099152000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768200.247881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768200.247885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768200.247888:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775768200.247893:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775768200.247897:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5661 00000100:00080000:2.0:1775768200.247901:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768200.247906:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768200.247919:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768200.247936:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099152128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768200.247944:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768200/real 1775768200] req@ffff8b23b9ea4e00 x1862022099151744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768200.247951:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768200.247958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768200.247961:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768200.247966:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5661 00000100:00100000:0.0:1775768200.247968:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022099151744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768200.247971:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768200.247980:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:0.0:1775768200.247983:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1775768200.247986:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775768200.247992:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768200.247998:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099151744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775768200.248000:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099152256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768200.248033:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099151872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768200.248058:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768200.248069:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768200/real 1775768200] req@ffff8b23b9ea4700 x1862022099151872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768200.248081:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099151872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768200.248089:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768200.248093:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768200.248097:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099151872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768200.248104:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099152000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768200.248125:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768200.248136:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768200/real 1775768200] req@ffff8b23b9ea4380 x1862022099152000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768200.248151:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099152000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768200.248157:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768200.248159:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768200.248162:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099152000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768200.248166:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099152128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768200.248182:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768200.248188:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768200/real 1775768200] req@ffff8b23b9ea7100 x1862022099152128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768200.248201:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099152128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768200.248206:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768200.248208:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768200.248210:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099152128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768200.248213:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099152256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768200.248224:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768200.248229:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768200/real 1775768200] req@ffff8b23b9ea6300 x1862022099152256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768200.248237:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099152256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768200.248242:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768200.248244:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768200.248245:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099152256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768201.143682:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768202.167737:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768203.191648:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768204.215657:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768205.239715:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768205.367597:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768205.367608:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768205.367613:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768205.367624:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5666 00000100:00080000:1.0:1775768205.367638:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768205.367645:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768205.367696:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099152384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768205.367730:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768205.367733:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768205.367734:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768205.367737:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5666 00000100:00080000:1.0:1775768205.367739:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768205.367742:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768205.367752:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768205.367760:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099152512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768205.367769:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768205.367772:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768205.367774:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768205.367801:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5666 00000100:00100000:0.0:1775768205.367806:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099152384:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775768205.367806:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768205.367811:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768205.367816:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768205.367830:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099152640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768205.367838:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768205.367841:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768205.367843:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768205.367846:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5666 00000100:00080000:1.0:1775768205.367849:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768205.367851:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768205.367855:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768205.367863:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099152768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768205.367869:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768205.367872:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768205.367873:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768205.367875:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5666 00000100:00080000:1.0:1775768205.367880:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768205.367883:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768205.367886:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768205.367894:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775768205.367895:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099152896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768205.367955:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768205/real 1775768205] req@ffff8b23864fea00 x1862022099152384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768205.367985:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099152384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768205.367998:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768205.368003:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768205.368007:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099152384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768205.368016:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099152512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768205.368036:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768205.368047:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768205/real 1775768205] req@ffff8b23864fd180 x1862022099152512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768205.368055:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099152512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768205.368061:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768205.368063:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768205.368065:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099152512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768205.368087:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099152640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768205.368099:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768205.368105:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768205/real 1775768205] req@ffff8b23864fce00 x1862022099152640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768205.368113:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099152640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768205.368124:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768205.368126:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768205.368128:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099152640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768205.368132:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099152768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768205.368143:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768205.368153:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768205/real 1775768205] req@ffff8b23864fe680 x1862022099152768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768205.368165:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099152768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768205.368173:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768205.368176:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768205.368180:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099152768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768205.368187:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099152896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768205.368207:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768205.368219:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768205/real 1775768205] req@ffff8b23864fca80 x1862022099152896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768205.368232:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099152896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768205.368240:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768205.368243:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768205.368247:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099152896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768206.263627:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768207.287637:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768208.311602:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768209.335588:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768210.359648:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768210.487524:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768210.487532:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768210.487537:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768210.487548:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5671 00000100:00080000:1.0:1775768210.487558:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768210.487563:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768210.487596:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099153024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768210.487621:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768210.487624:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768210.487626:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768210.487629:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5671 00000100:00080000:1.0:1775768210.487633:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768210.487636:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768210.487645:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768210.487656:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099153152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768210.487666:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768210.487669:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775768210.487672:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099153024:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775768210.487672:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768210.487680:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5671 00000100:00080000:1.0:1775768210.487683:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768210.487686:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768210.487691:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768210.487712:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099153280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768210.487721:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768210.487724:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768210.487756:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768210.487761:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5671 00000100:00080000:1.0:1775768210.487764:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768210.487767:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768210.487773:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768210.487774:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775768210.487788:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099153408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768210.487797:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768210.487800:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768210.487801:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768210.487805:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5671 00000100:00080000:1.0:1775768210.487809:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768210.487821:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768210.487825:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768210.487829:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768210/real 1775768210] req@ffff8b23864fdf80 x1862022099153024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768210.487839:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099153536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768210.487850:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099153024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768210.487876:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768210.487881:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768210.487884:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099153024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768210.487920:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099153152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768210.487942:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768210.487970:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768210/real 1775768210] req@ffff8b23864ff800 x1862022099153152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768210.488001:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099153152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768210.488010:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768210.488013:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768210.488017:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099153152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768210.488036:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099153280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768210.488055:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768210.488078:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768210/real 1775768210] req@ffff8b23864fed80 x1862022099153280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768210.488090:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099153280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768210.488111:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768210.488114:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768210.488117:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099153280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768210.488122:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099153408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768210.488151:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768210.488160:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768210/real 1775768210] req@ffff8b23864fc000 x1862022099153408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768210.488187:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099153408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768210.488195:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768210.488199:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768210.488202:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099153408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768210.488207:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099153536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768210.488235:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768210.488259:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768210/real 1775768210] req@ffff8b23864fc380 x1862022099153536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768210.488277:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099153536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768210.488294:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768210.488297:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768210.488301:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099153536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768211.383617:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768212.407588:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768213.431647:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768214.455611:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768215.479618:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768215.607602:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768215.607610:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768215.607614:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768215.607624:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5677 00000100:00080000:3.0:1775768215.607639:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768215.607645:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768215.607702:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099153664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768215.607731:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768215.607734:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768215.607736:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768215.607740:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5677 00000100:00080000:3.0:1775768215.607743:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768215.607747:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768215.607756:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768215.607771:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099153792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768215.607783:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768215.607786:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768215.607788:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768215.607795:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5677 00000100:00080000:3.0:1775768215.607798:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768215.607802:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768215.607806:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099153664:192.168.202.128@tcp:250:kworker.0 00000008:00100000:3.0:1775768215.607807:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768215.607822:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099153920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768215.607831:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768215.607834:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768215.607836:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768215.607840:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5677 00000100:00080000:3.0:1775768215.607843:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768215.607847:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768215.607851:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768215.607861:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099154048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768215.607870:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768215.607873:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768215.607874:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768215.607878:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5677 00000100:00080000:3.0:1775768215.607881:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768215.607884:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768215.607889:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768215.607895:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768215.607900:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099154176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768215.607951:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768215/real 1775768215] req@ffff8b23864f3480 x1862022099153664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768215.607972:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099153664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768215.607983:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768215.607988:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768215.607991:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099153664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768215.608020:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099153792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768215.608045:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768215.608057:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768215/real 1775768215] req@ffff8b23864f2a00 x1862022099153792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768215.608075:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099153792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768215.608082:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768215.608085:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768215.608089:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099153792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768215.608096:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099153920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768215.608116:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768215.608126:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768215/real 1775768215] req@ffff8b23864f0000 x1862022099153920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768215.608137:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099153920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768215.608144:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768215.608148:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768215.608151:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099153920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768215.608157:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099154048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768215.608177:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768215.608187:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768215/real 1775768215] req@ffff8b23864f0380 x1862022099154048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768215.608199:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099154048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768215.608211:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768215.608215:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768215.608218:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099154048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768215.608224:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099154176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768215.608241:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768215.608251:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768215/real 1775768215] req@ffff8b23864f3b80 x1862022099154176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768215.608261:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099154176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768215.608268:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768215.608271:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768215.608274:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099154176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768216.503664:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768217.527647:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768218.551705:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768219.575635:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768220.599693:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768220.727583:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768220.727597:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768220.727602:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768220.727613:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5682 00000100:00080000:1.0:1775768220.727638:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768220.727647:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768220.727696:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099154304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768220.727733:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768220.727740:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768220.727742:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768220.727747:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5682 00000100:00080000:1.0:1775768220.727752:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768220.727757:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768220.727768:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768220.727780:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099154304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775768220.727785:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099154432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768220.727803:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768220.727806:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768220.727808:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768220.727814:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5682 00000100:00080000:1.0:1775768220.727818:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768220.727822:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768220.727828:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768220.727842:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099154560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768220.727852:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775768220.727855:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775768220.727859:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768220.727862:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768220.727867:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5682 00000100:00080000:1.0:1775768220.727872:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768220.727876:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768220.727882:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768220.727894:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099154688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768220.727904:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768220/real 1775768220] req@ffff8b23864fc380 x1862022099154304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768220.727905:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768220.727914:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768220.727918:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768220.727924:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5682 00000100:00100000:0.0:1775768220.727928:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099154304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768220.727929:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768220.727938:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:0.0:1775768220.727942:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1775768220.727945:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775768220.727947:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768220.727952:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099154304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775768220.727962:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099154816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768220.727976:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099154432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768220.728001:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768220.728009:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768220/real 1775768220] req@ffff8b23864fc000 x1862022099154432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768220.728017:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099154432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768220.728022:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768220.728024:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768220.728026:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099154432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768220.728029:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099154560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768220.728039:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768220.728045:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768220/real 1775768220] req@ffff8b23864fed80 x1862022099154560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768220.728051:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099154560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768220.728055:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768220.728057:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768220.728059:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099154560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768220.728062:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099154688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768220.728079:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768220.728088:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768220/real 1775768220] req@ffff8b23864ff800 x1862022099154688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768220.728099:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099154688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768220.728106:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768220.728109:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768220.728112:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099154688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768220.728118:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099154816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768220.728136:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768220.728145:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768220/real 1775768220] req@ffff8b23864fdf80 x1862022099154816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768220.728156:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099154816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768220.728164:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768220.728167:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768220.728170:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099154816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768221.623610:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768222.647725:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768223.671711:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768224.695704:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768225.719686:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768225.847595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768225.847607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768225.847612:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768225.847624:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5687 00000100:00080000:2.0:1775768225.847643:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768225.847650:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768225.847700:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099154944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768225.847732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768225.847736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768225.847738:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768225.847742:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5687 00000100:00080000:2.0:1775768225.847746:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768225.847752:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768225.847763:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768225.847775:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099155072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768225.847790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768225.847793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768225.847795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775768225.847802:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099154944:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775768225.847804:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5687 00000100:00080000:2.0:1775768225.847813:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768225.847821:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768225.847826:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768225.847841:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099155200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768225.847852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768225.847856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768225.847858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768225.847862:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5687 00000100:00080000:2.0:1775768225.847865:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768225.847869:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768225.847874:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768225.847884:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775768225.847886:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099155328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768225.847895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768225.847898:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768225.847899:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768225.847903:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5687 00000100:00080000:2.0:1775768225.847906:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768225.847910:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768225.847914:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768225.847923:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099155456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768225.847948:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768225/real 1775768225] req@ffff8b23b9ea7480 x1862022099154944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768225.847969:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099154944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768225.847980:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768225.847984:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768225.847988:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099154944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768225.848010:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099155072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768225.848031:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768225.848042:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768225/real 1775768225] req@ffff8b23b9ea5880 x1862022099155072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768225.848053:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099155072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768225.848060:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768225.848064:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768225.848067:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099155072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768225.848073:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099155200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768225.848091:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768225.848100:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768225/real 1775768225] req@ffff8b23b9ea4000 x1862022099155200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768225.848112:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099155200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768225.848126:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768225.848129:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768225.848132:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099155200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768225.848138:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099155328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768225.848156:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768225.848164:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768225/real 1775768225] req@ffff8b23b9ea6a00 x1862022099155328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768225.848175:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099155328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768225.848183:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768225.848186:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768225.848189:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099155328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768225.848204:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099155456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768225.848221:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768225.848230:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768225/real 1775768225] req@ffff8b23b9ea4a80 x1862022099155456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768225.848241:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099155456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768225.848249:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768225.848252:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768225.848256:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099155456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768226.743719:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768227.767645:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768228.791694:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768229.815681:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768230.839681:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768230.967583:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768230.967595:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768230.967600:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768230.967609:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5692 00000100:00080000:2.0:1775768230.967622:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768230.967629:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768230.967683:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099155584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768230.967728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768230.967732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768230.967734:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768230.967742:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5692 00000100:00080000:2.0:1775768230.967746:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768230.967750:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768230.967762:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768230.967776:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099155712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768230.967789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768230.967793:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099155584:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775768230.967796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768230.967800:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768230.967807:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5692 00000100:00080000:2.0:1775768230.967810:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768230.967815:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768230.967820:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768230.967846:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099155840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768230.967856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768230.967860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768230.967862:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768230.967866:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5692 00000100:00080000:2.0:1775768230.967870:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768230.967877:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775768230.967879:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:2.0:1775768230.967884:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768230.967897:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099155968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768230.967907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768230.967910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768230.967912:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768230.967915:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5692 00000100:00080000:2.0:1775768230.967919:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768230.967923:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768230.967929:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768230.967936:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768230/real 1775768230] req@ffff8b23b9ea7b80 x1862022099155584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768230.967942:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099156096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768230.967967:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099155584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768230.967980:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768230.967985:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768230.967989:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099155584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768230.968019:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099155712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768230.968043:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768230.968055:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768230/real 1775768230] req@ffff8b23b9ea6680 x1862022099155712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768230.968068:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099155712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768230.968076:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768230.968080:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768230.968083:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099155712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768230.968089:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099155840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768230.968109:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768230.968119:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768230/real 1775768230] req@ffff8b23b9ea5f80 x1862022099155840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768230.968131:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099155840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768230.968139:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768230.968143:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768230.968146:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099155840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768230.968151:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099155968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768230.968172:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768230.968181:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768230/real 1775768230] req@ffff8b23b9ea6d80 x1862022099155968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768230.968192:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099155968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768230.968200:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768230.968203:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768230.968206:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099155968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768230.968217:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099156096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768230.968238:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768230.968247:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768230/real 1775768230] req@ffff8b23b9ea5c00 x1862022099156096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768230.968265:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099156096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768230.968273:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768230.968276:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768230.968279:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099156096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768231.863614:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768232.887725:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768233.911758:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768234.935659:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768235.959702:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768236.087552:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768236.087561:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768236.087565:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768236.087576:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5697 00000100:00080000:2.0:1775768236.087587:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768236.087594:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768236.087636:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022099156224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768236.087661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768236.087663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768236.087665:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768236.087667:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5697 00000100:00080000:2.0:1775768236.087669:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768236.087672:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768236.087680:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768236.087688:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099156352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768236.087696:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768236.087698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768236.087700:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768236.087702:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5697 00000100:00080000:2.0:1775768236.087705:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768236.087708:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768236.087711:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768236.087718:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099156224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775768236.087719:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099156480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768236.087731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768236.087734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768236.087735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768236.087738:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5697 00000100:00080000:2.0:1775768236.087740:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768236.087741:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768236.087744:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768236.087751:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099156608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768236.087756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768236.087758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768236.087759:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768236.087761:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5697 00000100:00080000:2.0:1775768236.087763:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768236.087766:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768236.087768:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768236.087775:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099156736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768236.087803:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768236.087863:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768236/real 1775768236] req@ffff8b23b9ea5500 x1862022099156224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768236.087882:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022099156224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768236.087892:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768236.087896:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768236.087900:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099156224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768236.087907:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099156352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768236.087929:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768236.087941:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768236/real 1775768236] req@ffff8b23b9ea5c00 x1862022099156352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768236.087958:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099156352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768236.087966:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768236.087968:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768236.087971:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099156352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768236.087998:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099156480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768236.088028:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768236.088043:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768236/real 1775768236] req@ffff8b23b9ea6d80 x1862022099156480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768236.088055:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099156480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768236.088062:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768236.088065:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768236.088068:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099156480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768236.088073:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099156608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768236.088091:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768236.088100:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768236/real 1775768236] req@ffff8b23b9ea5f80 x1862022099156608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768236.088110:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099156608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768236.088124:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768236.088126:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768236.088130:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099156608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768236.088134:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099156736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768236.088151:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768236.088160:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768236/real 1775768236] req@ffff8b23b9ea6680 x1862022099156736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768236.088170:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099156736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768236.088177:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768236.088180:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768236.088183:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099156736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768236.984703:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768238.007636:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768239.031657:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768240.055612:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768241.079620:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768241.207591:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768241.207598:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768241.207601:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768241.207609:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5702 00000100:00080000:3.0:1775768241.207618:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768241.207622:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768241.207651:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099156864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768241.207676:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768241.207679:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768241.207680:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768241.207682:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5702 00000100:00080000:3.0:1775768241.207684:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768241.207686:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768241.207723:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768241.207738:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099156992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768241.207749:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768241.207752:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768241.207753:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768241.207757:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5702 00000100:00080000:3.0:1775768241.207761:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768241.207765:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768241.207769:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768241.207780:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099157120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768241.207789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768241.207792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768241.207794:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768241.207798:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5702 00000100:00080000:3.0:1775768241.207801:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768241.207804:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768241.207807:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768241.207814:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099157248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768241.207830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768241.207832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768241.207833:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768241.207836:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5702 00000100:00080000:3.0:1775768241.207838:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768241.207840:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768241.207843:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768241.207848:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099157376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768241.207933:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099156864:192.168.202.128@tcp:250:kworker.0 00000800:00000100:0.0:1775768241.207982:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768241.208010:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768241/real 1775768241] req@ffff8b23864f1180 x1862022099156864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768241.208024:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099156864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768241.208032:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768241.208037:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768241.208040:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099156864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768241.208046:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099156992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768241.208067:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768241.208077:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768241/real 1775768241] req@ffff8b23864f2d80 x1862022099156992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768241.208088:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099156992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768241.208094:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768241.208096:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768241.208099:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099156992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768241.208103:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099157120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768241.208119:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768241.208127:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768241/real 1775768241] req@ffff8b23864f2680 x1862022099157120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768241.208136:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099157120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768241.208143:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768241.208146:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768241.208148:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099157120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768241.208153:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099157248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768241.208171:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768241.208181:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768241/real 1775768241] req@ffff8b23864f1c00 x1862022099157248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768241.208189:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099157248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768241.208194:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768241.208196:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768241.208198:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099157248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768241.208201:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099157376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768241.208214:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768241.208219:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768241/real 1775768241] req@ffff8b23864f0700 x1862022099157376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768241.208225:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099157376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768241.208229:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768241.208231:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768241.208232:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099157376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768242.103632:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768243.127680:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768244.151647:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768245.175685:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768246.199786:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768246.327555:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768246.327564:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768246.327568:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768246.327577:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5707 00000100:00080000:0.0:1775768246.327587:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768246.327637:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768246.327683:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099157504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768246.327712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768246.327715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768246.327716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768246.327720:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5707 00000100:00080000:0.0:1775768246.327723:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768246.327727:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768246.327735:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768246.327749:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099157632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768246.327756:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099157504:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775768246.327761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768246.327765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768246.327766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768246.327780:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5707 00000100:00080000:0.0:1775768246.327784:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768246.327787:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768246.327791:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768246.327801:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099157760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768246.327811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768246.327814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768246.327815:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768246.327817:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5707 00000100:00080000:0.0:1775768246.327820:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768246.327822:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768246.327825:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768246.327832:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099157888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768246.327838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768246.327840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768246.327841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768246.327844:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5707 00000100:00080000:0.0:1775768246.327845:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768246.327847:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768246.327849:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768246.327857:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099157632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768246.327860:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099158016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768246.327891:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099157760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768246.327910:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099157888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768246.329499:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099158016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768247.223667:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768248.247623:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768249.271600:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775768249.399728:0:2366:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775768249.399762:0:2366:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775768249.399783:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775768249.399828:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775768249.399838:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775768249.399844:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775768249.399851:0:2366:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:3.0:1775768249.399952:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768246/real 1775768249] req@ffff8b2382daf480 x1862022099158016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768249.399978:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099158016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768249.399989:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768249.400000:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768249.400005:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099158016:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768249.400014:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768246/real 1775768249] req@ffff8b2382dad500 x1862022099157632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768249.400026:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099157632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768249.400033:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768249.400036:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768249.400039:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099157632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768249.400045:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768246/real 1775768249] req@ffff8b2382daea00 x1862022099157760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768249.400074:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099157760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768249.400082:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768249.400085:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768249.400089:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099157760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768249.400095:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768246/real 1775768249] req@ffff8b2382daed80 x1862022099157888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768249.400105:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099157888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768249.400111:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768249.400115:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768249.400117:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099157888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768249.400123:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768246/real 1775768249] req@ffff8b2382dae680 x1862022099157504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768249.400138:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099157504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768249.400144:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768249.400146:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768249.400149:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099157504:192.168.202.128@tcp:250:kworker.0 00010000:00010000:0.0:1775768250.295634:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768251.319593:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768251.447559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768251.447569:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768251.447581:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768251.447590:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5712 00000100:00080000:0.0:1775768251.447599:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768251.447605:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768251.447638:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022099158144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768251.447665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768251.447669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768251.447670:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768251.447674:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5712 00000100:00080000:0.0:1775768251.447677:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768251.447680:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768251.447689:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768251.447699:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022099158272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768251.447711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768251.447714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768251.447716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768251.447720:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5712 00000100:00080000:0.0:1775768251.447724:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768251.447727:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768251.447732:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768251.447743:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022099158400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768251.447753:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099158144:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775768251.447791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768251.447797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768251.447799:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768251.447804:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5712 00000100:00080000:0.0:1775768251.447808:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768251.447813:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768251.447818:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768251.447832:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022099158528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768251.447841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768251.447845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768251.447846:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768251.447850:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5712 00000100:00080000:0.0:1775768251.447854:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768251.447858:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768251.447863:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775768251.447869:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768251.447880:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099158656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768251.447943:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768251/real 1775768251] req@ffff8b2382dac700 x1862022099158144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768251.447966:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022099158144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768251.447977:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768251.447982:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768251.447986:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099158144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768251.447995:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099158272:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768251.448020:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768251.448033:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768251/real 1775768251] req@ffff8b2382dace00 x1862022099158272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768251.448044:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022099158272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768251.448053:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768251.448057:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768251.448060:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099158272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768251.448298:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099158400:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768251.448369:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768251.448393:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768251/real 1775768251] req@ffff8b2382dadf80 x1862022099158400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768251.448410:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022099158400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768251.448420:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768251.448423:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768251.448426:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099158400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768251.448434:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099158528:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768251.448487:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768251.448500:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768251/real 1775768251] req@ffff8b2382dafb80 x1862022099158528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768251.448511:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022099158528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768251.448518:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768251.448521:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768251.448525:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099158528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768251.448531:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099158656:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768251.448549:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768251.448559:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768251/real 1775768251] req@ffff8b2382dae680 x1862022099158656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768251.448569:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099158656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768251.448575:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768251.448578:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768251.448580:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099158656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768252.343613:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768253.367648:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768254.391582:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768255.415636:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768256.439591:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768256.567574:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768256.567584:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768256.567589:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768256.567601:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5717 00000100:00080000:3.0:1775768256.567614:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768256.567621:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768256.567669:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099158784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768256.567702:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768256.567705:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768256.567707:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768256.567713:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5717 00000100:00080000:3.0:1775768256.567718:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768256.567722:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768256.567733:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768256.567747:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099158912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768256.567748:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099158784:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768256.567775:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768256.567779:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768256.567781:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768256.567785:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5717 00000100:00080000:3.0:1775768256.567789:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768256.567794:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768256.567799:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768256.567813:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099159040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768256.567822:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768256.567826:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768256.567827:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775768256.567833:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775768256.567834:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5717 00000100:00080000:3.0:1775768256.567842:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768256.567846:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768256.567852:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768256.567867:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099159168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768256.567877:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768256/real 1775768256] req@ffff8b23864f3100 x1862022099158784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768256.567880:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768256.567893:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768256.567900:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768256.567908:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5717 00000100:00100000:0.0:1775768256.567913:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099158784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768256.567915:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768256.567928:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:0.0:1775768256.567939:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1775768256.567941:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775768256.567949:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768256.567954:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099158784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768256.567962:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099159296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768256.567978:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099158912:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768256.567995:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768256.568004:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768256/real 1775768256] req@ffff8b23864f0a80 x1862022099158912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768256.568011:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099158912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768256.568016:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768256.568018:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768256.568020:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099158912:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768256.568024:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099159040:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768256.568034:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768256.568038:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768256/real 1775768256] req@ffff8b23864f1f80 x1862022099159040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768256.568048:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099159040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768256.568053:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768256.568054:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768256.568056:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099159040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768256.568059:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099159168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768256.568074:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768256.568083:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768256/real 1775768256] req@ffff8b23864f0700 x1862022099159168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768256.568093:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099159168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768256.568101:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768256.568104:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768256.568107:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099159168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768256.568113:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099159296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768256.568163:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768256.568174:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768256/real 1775768256] req@ffff8b23864f1c00 x1862022099159296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768256.568185:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099159296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768256.568196:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768256.568198:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768256.568200:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099159296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768257.463635:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768258.487574:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768259.511555:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768260.535659:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768261.559619:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768261.687566:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768261.687572:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768261.687576:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768261.687584:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5723 00000100:00080000:3.0:1775768261.687593:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768261.687598:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768261.687629:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099159424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768261.687651:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768261.687653:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768261.687654:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768261.687658:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5723 00000100:00080000:3.0:1775768261.687660:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768261.687662:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768261.687669:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768261.687678:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099159552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768261.687687:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768261.687693:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768261.687694:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775768261.687699:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099159424:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768261.687699:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5723 00000100:00080000:3.0:1775768261.687703:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768261.687708:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768261.687711:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768261.687719:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099159680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768261.687725:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768261.687727:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768261.687728:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768261.687730:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5723 00000100:00080000:3.0:1775768261.687732:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768261.687734:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768261.687736:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768261.687741:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099159808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768261.687747:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768261.687749:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768261.687750:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768261.687752:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5723 00000100:00080000:3.0:1775768261.687754:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775768261.687757:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775768261.687760:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768261.687763:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768261.687769:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099159936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768261.687798:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768261/real 1775768261] req@ffff8b23864f2680 x1862022099159424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768261.687830:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099159424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768261.687839:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768261.687843:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768261.687847:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099159424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768261.687854:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099159552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768261.687874:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768261.687885:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768261/real 1775768261] req@ffff8b23864f2d80 x1862022099159552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768261.687896:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099159552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768261.687903:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768261.687906:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768261.687909:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099159552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768261.687927:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099159680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768261.687944:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768261.687954:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768261/real 1775768261] req@ffff8b23864f1180 x1862022099159680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768261.687965:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099159680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768261.687972:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768261.687975:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768261.687978:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099159680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768261.687983:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099159808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768261.688001:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768261.688010:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768261/real 1775768261] req@ffff8b23864f3b80 x1862022099159808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768261.688020:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099159808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768261.688027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768261.688029:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768261.688032:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099159808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768261.688037:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099159936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768261.688054:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768261.688062:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768261/real 1775768261] req@ffff8b23864f0380 x1862022099159936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768261.688073:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099159936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768261.688079:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768261.688082:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768261.688085:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099159936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768262.583677:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768263.607665:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768264.631644:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768265.655685:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768266.679639:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768266.807572:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768266.807579:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768266.807582:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768266.807589:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5728 00000100:00080000:1.0:1775768266.807597:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768266.807602:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768266.807634:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099160064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768266.807656:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768266.807658:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768266.807659:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768266.807662:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5728 00000100:00080000:1.0:1775768266.807664:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768266.807666:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768266.807673:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768266.807687:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099160192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768266.807699:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768266.807703:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775768266.807706:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099160064:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775768266.807706:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768266.807712:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5728 00000100:00080000:1.0:1775768266.807716:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768266.807719:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768266.807724:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768266.807736:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099160320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768266.807743:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768266.807747:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768266.807749:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768266.807752:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5728 00000100:00080000:1.0:1775768266.807764:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768266.807767:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775768266.807772:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:1.0:1775768266.807781:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768266.807794:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099160448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768266.807806:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768266/real 1775768266] req@ffff8b23864fca80 x1862022099160064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768266.807814:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768266.807817:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768266.807819:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775768266.807823:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099160064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768266.807825:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5728 00000100:00080000:1.0:1775768266.807832:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768266.807835:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768266.807835:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:0.0:1775768266.807842:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000008:00100000:1.0:1775768266.807843:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768266.807846:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099160064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775768266.807855:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099160576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768266.807872:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099160192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768266.807895:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768266.807907:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768266/real 1775768266] req@ffff8b23864fe680 x1862022099160192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768266.807915:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099160192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768266.807925:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768266.807928:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768266.807932:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099160192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768266.807937:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099160320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768266.807956:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768266.807967:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768266/real 1775768266] req@ffff8b23864fce00 x1862022099160320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768266.807979:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099160320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768266.807986:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768266.807989:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768266.807992:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099160320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768266.807998:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099160448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768266.808017:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768266.808026:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768266/real 1775768266] req@ffff8b23864fd180 x1862022099160448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768266.808037:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099160448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768266.808044:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768266.808047:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768266.808050:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099160448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768266.808056:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099160576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768266.808083:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768266.808093:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768266/real 1775768266] req@ffff8b23864fea00 x1862022099160576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768266.808104:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099160576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768266.808111:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768266.808114:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768266.808117:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099160576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768267.703660:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768268.727586:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768269.751605:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768270.775641:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768271.799605:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768271.927597:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768271.927611:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768271.927616:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768271.927629:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5733 00000100:00080000:1.0:1775768271.927650:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768271.927659:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768271.927707:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099160704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768271.927734:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768271.927737:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768271.927738:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768271.927741:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5733 00000100:00080000:1.0:1775768271.927744:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768271.927747:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768271.927757:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768271.927769:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099160832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768271.927774:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099160704:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775768271.927781:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768271.927784:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768271.927785:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768271.927789:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5733 00000100:00080000:1.0:1775768271.927791:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768271.927794:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768271.927796:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768271.927809:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099160960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768271.927815:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768271.927818:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768271.927819:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768271.927822:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5733 00000100:00080000:1.0:1775768271.927840:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768271.927842:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768271.927845:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768271.927851:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775768271.927854:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099161088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768271.927861:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768271.927864:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768271.927865:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768271.927869:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5733 00000100:00080000:1.0:1775768271.927873:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768271.927876:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768271.927878:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768271.927885:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099161216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768271.927898:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768271/real 1775768271] req@ffff8b23864fdc00 x1862022099160704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768271.927913:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099160704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768271.927920:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768271.927924:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768271.927927:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099160704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768271.927947:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099160832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768271.927962:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768271.927979:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768271/real 1775768271] req@ffff8b23864fd500 x1862022099160832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768271.927990:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099160832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768271.927998:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768271.928001:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768271.928004:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099160832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768271.928010:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099160960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768271.928028:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768271.928038:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768271/real 1775768271] req@ffff8b23864ff100 x1862022099160960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768271.928045:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099160960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768271.928050:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768271.928052:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768271.928055:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099160960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768271.928058:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099161088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768271.928079:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768271.928086:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768271/real 1775768271] req@ffff8b23864fe300 x1862022099161088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768271.928092:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099161088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768271.928097:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768271.928099:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768271.928101:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099161088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768271.928104:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099161216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768271.928114:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768271.928118:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768271/real 1775768271] req@ffff8b23864ffb80 x1862022099161216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768271.928124:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099161216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768271.928129:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768271.928149:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768271.928152:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099161216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768272.823565:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768273.847562:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768274.871634:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768275.895624:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768276.919621:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768277.047561:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768277.047571:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768277.047575:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768277.047585:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5738 00000100:00080000:0.0:1775768277.047595:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768277.047601:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768277.047634:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099161344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768277.047687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768277.047690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768277.047692:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768277.047696:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5738 00000100:00080000:0.0:1775768277.047699:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768277.047703:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768277.047710:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768277.047722:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099161472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768277.047741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775768277.047743:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099161344:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775768277.047745:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768277.047749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768277.047754:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5738 00000100:00080000:0.0:1775768277.047757:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768277.047761:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768277.047766:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768277.047778:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099161600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768277.047789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768277.047791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768277.047793:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768277.047795:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5738 00000100:00080000:0.0:1775768277.047798:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768277.047800:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768277.047803:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768277.047810:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099161728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768277.047815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768277.047820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:3.0:1775768277.047820:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775768277.047824:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768277.047827:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5738 00000100:00080000:0.0:1775768277.047829:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768277.047831:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768277.047833:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768277.047840:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099161856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768277.047866:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768277/real 1775768277] req@ffff8b2382daed80 x1862022099161344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768277.047880:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099161344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768277.047887:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768277.047889:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768277.047893:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099161344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768277.047928:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099161472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768277.047951:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768277.047962:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768277/real 1775768277] req@ffff8b2382daea00 x1862022099161472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768277.047973:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099161472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768277.047980:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768277.047984:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768277.047987:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099161472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768277.047992:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099161600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768277.048011:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768277.048020:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768277/real 1775768277] req@ffff8b2382dad500 x1862022099161600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768277.048029:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099161600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768277.048036:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768277.048039:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768277.048042:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099161600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768277.048047:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099161728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768277.048064:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768277.048074:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768277/real 1775768277] req@ffff8b2382daf480 x1862022099161728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768277.048101:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099161728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768277.048108:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768277.048111:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768277.048113:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099161728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768277.048118:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099161856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768277.048145:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768277.048155:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768277/real 1775768277] req@ffff8b2382daca80 x1862022099161856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768277.048166:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099161856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768277.048173:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768277.048176:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768277.048179:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099161856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768277.943582:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768278.967703:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768279.991651:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768281.015642:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768282.039629:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768282.167561:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768282.167568:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768282.167571:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768282.167580:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5743 00000100:00080000:0.0:1775768282.167590:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768282.167618:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768282.167647:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022099161984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768282.167668:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768282.167671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768282.167672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768282.167675:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5743 00000100:00080000:0.0:1775768282.167678:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768282.167682:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768282.167690:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768282.167703:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022099162112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768282.167716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775768282.167716:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099161984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775768282.167723:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768282.167726:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768282.167730:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5743 00000100:00080000:0.0:1775768282.167734:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768282.167738:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768282.167743:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768282.167756:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099162240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768282.167771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768282.167774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768282.167776:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768282.167779:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5743 00000100:00080000:0.0:1775768282.167783:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768282.167786:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768282.167791:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1775768282.167793:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768282.167805:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022099162368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768282.167812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768282.167814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768282.167815:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768282.167817:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5743 00000100:00080000:0.0:1775768282.167821:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768282.167823:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768282.167826:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768282.167834:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099162496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768282.167839:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768282/real 1775768282] req@ffff8b2382daf100 x1862022099161984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768282.167864:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022099161984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768282.167874:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768282.167879:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768282.167882:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099161984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768282.167906:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099162112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768282.167926:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768282.167936:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768282/real 1775768282] req@ffff8b2382dac000 x1862022099162112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768282.167947:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022099162112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768282.167954:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768282.167957:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768282.167960:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099162112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768282.167965:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099162240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768282.167983:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768282.167992:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768282/real 1775768282] req@ffff8b2382dac380 x1862022099162240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768282.168002:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099162240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768282.168013:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768282.168016:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768282.168019:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099162240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768282.168025:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099162368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768282.168056:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768282.168066:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768282/real 1775768282] req@ffff8b2382dad880 x1862022099162368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768282.168075:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022099162368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768282.168082:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768282.168085:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768282.168088:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099162368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768282.168092:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099162496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768282.168106:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768282.168114:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768282/real 1775768282] req@ffff8b2382daca80 x1862022099162496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768282.168125:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099162496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768282.168132:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768282.168134:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768282.168138:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099162496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768283.063635:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768284.087641:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768285.111967:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768286.138186:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768287.159873:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768287.287558:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768287.287565:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768287.287569:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768287.287577:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5748 00000100:00080000:0.0:1775768287.287592:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768287.287598:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768287.287632:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099162624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768287.287663:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768287.287667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768287.287669:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768287.287673:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5748 00000100:00080000:0.0:1775768287.287676:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768287.287680:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768287.287688:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768287.287700:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099162752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768287.287712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768287.287715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768287.287717:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768287.287722:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5748 00000100:00080000:0.0:1775768287.287726:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768287.287730:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768287.287734:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768287.287747:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099162880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768287.287757:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099162624:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775768287.287782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768287.287785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768287.287787:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768287.287791:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5748 00000100:00080000:0.0:1775768287.287794:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768287.287798:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768287.287803:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768287.287817:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099163008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768287.287825:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768287.287828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768287.287829:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768287.287832:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5748 00000100:00080000:0.0:1775768287.287836:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768287.287840:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768287.287846:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768287.287857:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099163136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775768287.287866:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768287.287932:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768287/real 1775768287] req@ffff8b2382daf480 x1862022099162624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768287.287954:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099162624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768287.287962:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768287.287966:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768287.287968:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099162624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768287.287975:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099162752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768287.288008:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768287.288020:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768287/real 1775768287] req@ffff8b2382dad500 x1862022099162752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768287.288032:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099162752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768287.288040:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768287.288044:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768287.288047:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099162752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768287.288076:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099162880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768287.288093:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768287.288101:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768287/real 1775768287] req@ffff8b2382daea00 x1862022099162880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768287.288108:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099162880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768287.288113:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768287.288116:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768287.288117:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099162880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768287.288122:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099163008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768287.288276:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768287.288302:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768287/real 1775768287] req@ffff8b2382daed80 x1862022099163008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768287.288320:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099163008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768287.288330:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768287.288334:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768287.288338:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099163008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768287.288347:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099163136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768287.288371:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768287.288379:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768287/real 1775768287] req@ffff8b2382dae680 x1862022099163136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768287.288412:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099163136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768287.288419:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768287.288422:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768287.288425:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099163136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768288.183657:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768289.207643:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768290.231587:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768291.255633:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768292.279556:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768292.407608:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768292.407616:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768292.407620:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768292.407629:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5753 00000100:00080000:2.0:1775768292.407640:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768292.407645:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768292.407689:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099163264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768292.407711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768292.407714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768292.407715:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768292.407718:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5753 00000100:00080000:2.0:1775768292.407720:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768292.407723:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768292.407730:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768292.407738:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099163392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768292.407754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768292.407757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768292.407760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768292.407765:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5753 00000100:00080000:2.0:1775768292.407768:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768292.407775:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768292.407777:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099163264:192.168.202.128@tcp:250:kworker.0 00000008:00100000:2.0:1775768292.407782:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768292.407798:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099163520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768292.407811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768292.407814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768292.407816:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768292.407820:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5753 00000100:00080000:2.0:1775768292.407824:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768292.407828:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768292.407833:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768292.407844:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099163648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768292.407853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768292.407857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768292.407858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768292.407864:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5753 00000800:00000100:3.0:1775768292.407866:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775768292.407880:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768292.407883:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768292.407889:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768292.407904:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099163776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768292.407951:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768292/real 1775768292] req@ffff8b23b9ea7b80 x1862022099163264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768292.407977:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099163264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768292.407999:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768292.408004:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768292.408009:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099163264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768292.408017:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099163392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768292.408042:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768292.408054:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768292/real 1775768292] req@ffff8b23b9ea4a80 x1862022099163392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768292.408071:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099163392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768292.408079:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768292.408082:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768292.408090:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099163392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768292.409271:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099163520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768292.409319:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768292.409335:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768292/real 1775768292] req@ffff8b23b9ea6a00 x1862022099163520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768292.409348:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099163520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768292.409354:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768292.409357:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768292.409359:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099163520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768292.409364:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099163648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768292.409374:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768292.409379:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768292/real 1775768292] req@ffff8b23b9ea4000 x1862022099163648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768292.409386:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099163648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768292.409394:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768292.409396:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768292.409398:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099163648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768292.409401:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099163776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768292.409409:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768292.409413:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768292/real 1775768292] req@ffff8b23b9ea5880 x1862022099163776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768292.409419:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099163776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768292.409423:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768292.409425:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768292.409426:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099163776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768293.303582:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768294.327634:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768295.351631:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768296.375627:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768297.399653:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768297.527587:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768297.527600:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768297.527605:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768297.527618:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5758 00000100:00080000:3.0:1775768297.527632:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768297.527641:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768297.527688:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099163904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768297.527730:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768297.527734:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768297.527736:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768297.527740:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5758 00000100:00080000:3.0:1775768297.527744:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768297.527748:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768297.527760:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768297.527773:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099164032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768297.527778:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099163904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768297.527790:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768297.527794:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768297.527796:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768297.527801:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5758 00000100:00080000:3.0:1775768297.527804:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768297.527809:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768297.527813:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768297.527827:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099164160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768297.527837:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768297.527840:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768297.527842:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768297.527846:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5758 00000100:00080000:3.0:1775768297.527850:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768297.527853:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775768297.527859:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:3.0:1775768297.527859:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768297.527873:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099164288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768297.527896:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768297.527899:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768297.527901:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768297.527905:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5758 00000100:00080000:3.0:1775768297.527909:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768297.527912:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768297.527917:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768297.527930:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099164416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768297.527935:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768297/real 1775768297] req@ffff8b23864f0000 x1862022099163904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768297.527959:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099163904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768297.527971:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768297.527975:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768297.527979:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099163904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768297.528004:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099164032:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768297.528027:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768297.528040:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768297/real 1775768297] req@ffff8b23864f2a00 x1862022099164032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768297.528052:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099164032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768297.528060:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768297.528063:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768297.528066:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099164032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768297.528073:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099164160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768297.528090:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768297.528097:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768297/real 1775768297] req@ffff8b23864f3480 x1862022099164160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768297.528115:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099164160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768297.528120:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768297.528122:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768297.528124:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099164160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768297.528127:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099164288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768297.528140:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768297.528146:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768297/real 1775768297] req@ffff8b23864f0380 x1862022099164288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768297.528152:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099164288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768297.528161:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768297.528162:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768297.528164:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099164288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768297.528167:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099164416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768297.528183:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768297.528192:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768297/real 1775768297] req@ffff8b23864f3b80 x1862022099164416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768297.528202:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099164416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768297.528215:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768297.528218:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768297.528220:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099164416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768298.423665:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768299.447603:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768300.471668:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768301.495672:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768302.519605:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768302.647569:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768302.647577:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768302.647581:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768302.647589:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5764 00000100:00080000:3.0:1775768302.647598:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768302.647603:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768302.647634:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099164544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768302.647662:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768302.647666:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768302.647667:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768302.647671:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5764 00000100:00080000:3.0:1775768302.647674:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768302.647678:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768302.647687:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768302.647701:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099164672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768302.647714:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768302.647718:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768302.647719:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768302.647723:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5764 00000100:00080000:3.0:1775768302.647727:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768302.647732:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768302.647737:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768302.647751:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099164544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768302.647751:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099164800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768302.647774:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768302.647777:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768302.647779:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768302.647784:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5764 00000100:00080000:3.0:1775768302.647788:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768302.647792:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768302.647797:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768302.647809:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099164928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768302.647819:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768302.647823:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768302.647824:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768302.647828:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5764 00000100:00080000:3.0:1775768302.647832:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768302.647851:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768302.647861:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768302.647866:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768302.647880:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099165056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768302.647933:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768302/real 1775768302] req@ffff8b23864f1180 x1862022099164544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768302.647953:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099164544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768302.647964:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768302.647968:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768302.647973:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099164544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768302.647981:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099164672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768302.648005:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768302.648015:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768302/real 1775768302] req@ffff8b23864f2d80 x1862022099164672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768302.648023:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099164672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768302.648028:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768302.648031:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768302.648032:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099164672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768302.648055:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099164800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768302.648066:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768302.648072:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768302/real 1775768302] req@ffff8b23864f2680 x1862022099164800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768302.648079:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099164800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768302.648084:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768302.648086:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768302.648087:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099164800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768302.648091:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099164928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768302.648101:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768302.648106:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768302/real 1775768302] req@ffff8b23864f1c00 x1862022099164928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768302.648112:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099164928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768302.648117:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768302.648119:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768302.648120:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099164928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768302.648123:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099165056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768302.648134:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768302.648141:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768302/real 1775768302] req@ffff8b23864f0700 x1862022099165056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768302.648151:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099165056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768302.648158:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768302.648161:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768302.648164:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099165056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768303.543637:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768304.567672:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768305.591688:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768306.615685:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768307.639671:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768307.767546:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768307.767555:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768307.767558:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768307.767565:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5769 00000100:00080000:3.0:1775768307.767575:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768307.767579:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768307.767609:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099165184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768307.767634:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768307.767636:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768307.767637:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768307.767640:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5769 00000100:00080000:3.0:1775768307.767641:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768307.767644:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768307.767650:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768307.767659:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099165312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768307.767667:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768307.767669:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768307.767670:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768307.767673:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5769 00000100:00080000:3.0:1775768307.767674:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768307.767677:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768307.767679:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768307.767682:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099165184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768307.767687:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099165440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768307.767693:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768307.767695:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768307.767696:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768307.767698:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5769 00000100:00080000:3.0:1775768307.767701:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768307.767703:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768307.767706:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768307.767713:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099165568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768307.767718:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768307.767720:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768307.767721:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768307.767723:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5769 00000100:00080000:3.0:1775768307.767726:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768307.767728:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768307.767730:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768307.767735:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099165696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768307.767750:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768307.767816:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768307/real 1775768307] req@ffff8b23864f1f80 x1862022099165184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768307.767833:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099165184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768307.767842:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768307.767845:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768307.767849:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099165184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768307.767871:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099165312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768307.767894:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768307.767904:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768307/real 1775768307] req@ffff8b23864f0a80 x1862022099165312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768307.767912:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099165312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768307.767917:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768307.767919:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768307.767921:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099165312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768307.767925:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099165440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768307.767939:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768307.767947:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768307/real 1775768307] req@ffff8b23864f3100 x1862022099165440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768307.767958:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099165440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768307.767965:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768307.767967:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768307.767970:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099165440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768307.767975:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099165568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768307.767995:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768307.768004:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768307/real 1775768307] req@ffff8b23864f0700 x1862022099165568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768307.768015:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099165568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768307.768022:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768307.768025:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768307.768028:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099165568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768307.768034:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099165696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768307.768053:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768307.768062:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768307/real 1775768307] req@ffff8b23864f1c00 x1862022099165696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768307.768073:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099165696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768307.768081:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768307.768084:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768307.768086:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099165696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768308.663622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768309.687594:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768310.711665:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768311.735597:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768312.759598:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768312.887579:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768312.887589:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768312.887594:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768312.887604:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5774 00000100:00080000:0.0:1775768312.887616:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768312.887622:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768312.887675:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022099165824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768312.887713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768312.887717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768312.887719:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768312.887723:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5774 00000100:00080000:0.0:1775768312.887727:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768312.887732:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768312.887764:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099165824:192.168.202.128@tcp:250:kworker.0 00000008:00100000:0.0:1775768312.887772:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768312.887794:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022099165952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768312.887804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768312.887807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768312.887808:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768312.887811:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5774 00000100:00080000:0.0:1775768312.887815:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768312.887817:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768312.887822:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768312.887832:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022099166080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768312.887837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768312.887840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768312.887841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768312.887844:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5774 00000100:00080000:0.0:1775768312.887846:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768312.887849:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768312.887851:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768312.887859:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022099166208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768312.887865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768312.887868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768312.887869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768312.887872:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5774 00000100:00080000:0.0:1775768312.887874:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768312.887877:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768312.887877:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099165952:192.168.202.128@tcp:38:kworker.0 00000008:00100000:0.0:1775768312.887884:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768312.887908:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099166080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768312.887923:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099166208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768312.888039:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099166336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768312.888111:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099166336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768313.783760:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768314.807625:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768315.831686:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775768315.959733:0:2363:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775768315.959775:0:2363:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775768315.959792:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775768315.959835:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775768315.959844:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775768315.959853:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775768315.959860:0:2363:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:3.0:1775768315.959941:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768312/real 1775768315] req@ffff8b2382dae680 x1862022099166336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768315.959974:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099166336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768315.959989:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768315.959995:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768315.959999:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099166336:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768315.960009:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768312/real 1775768315] req@ffff8b2382dadf80 x1862022099165952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768315.960022:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022099165952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768315.960030:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768315.960035:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768315.960038:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099165952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768315.960045:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768312/real 1775768315] req@ffff8b2382dace00 x1862022099166080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768315.960058:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022099166080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768315.960066:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768315.960070:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768315.960074:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099166080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768315.960082:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768312/real 1775768315] req@ffff8b2382dac700 x1862022099166208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768315.960094:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022099166208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768315.960102:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768315.960105:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768315.960108:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099166208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768315.960114:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768312/real 1775768315] req@ffff8b2382dafb80 x1862022099165824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768315.960125:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022099165824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768315.960132:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768315.960141:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768315.960145:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099165824:192.168.202.128@tcp:250:kworker.0 00010000:00010000:3.0:1775768316.855691:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768317.879603:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768318.007571:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768318.007583:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768318.007588:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768318.007598:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5779 00000100:00080000:0.0:1775768318.007612:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768318.007619:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768318.007661:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099166464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768318.007715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768318.007718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768318.007720:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768318.007723:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5779 00000100:00080000:0.0:1775768318.007726:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768318.007731:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768318.007746:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768318.007763:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099166592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768318.007777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768318.007783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768318.007785:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1775768318.007790:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099166464:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775768318.007793:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5779 00000100:00080000:0.0:1775768318.007800:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768318.007806:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768318.007811:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768318.007826:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099166720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768318.007836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768318.007840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768318.007842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768318.007847:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5779 00000100:00080000:0.0:1775768318.007860:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768318.007865:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768318.007869:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768318.007879:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099166848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768318.007886:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768318.007890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768318.007892:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768318.007896:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5779 00000100:00080000:0.0:1775768318.007900:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1775768318.007905:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775768318.007906:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768318.007914:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768318.007928:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099166976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768318.007984:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768318/real 1775768318] req@ffff8b2382daed80 x1862022099166464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768318.008009:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099166464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768318.008027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768318.008031:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768318.008036:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099166464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768318.008081:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099166592:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768318.008109:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768318.008122:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768318/real 1775768318] req@ffff8b2382daea00 x1862022099166592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768318.008135:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099166592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768318.008143:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768318.008147:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768318.008150:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099166592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768318.008155:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099166720:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768318.008184:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768318.008195:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768318/real 1775768318] req@ffff8b2382dad500 x1862022099166720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768318.008207:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099166720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768318.008214:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768318.008218:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768318.008222:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099166720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768318.008227:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099166848:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768318.008256:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768318.008266:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768318/real 1775768318] req@ffff8b2382daf480 x1862022099166848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768318.008275:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099166848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768318.008282:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768318.008286:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768318.008289:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099166848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768318.008294:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099166976:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768318.008314:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768318.008322:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768318/real 1775768318] req@ffff8b2382daca80 x1862022099166976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768318.008416:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099166976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768318.008425:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768318.008428:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768318.008431:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099166976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768318.903652:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768319.927642:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768320.951714:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768321.975661:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768322.999678:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768323.127593:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768323.127602:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768323.127607:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768323.127618:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5784 00000100:00080000:0.0:1775768323.127630:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768323.127654:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768323.127699:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022099167104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768323.127729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768323.127732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768323.127734:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768323.127737:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5784 00000100:00080000:0.0:1775768323.127739:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768323.127742:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768323.127750:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768323.127762:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099167232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768323.127775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768323.127778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768323.127780:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768323.127783:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5784 00000100:00080000:0.0:1775768323.127785:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768323.127790:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768323.127792:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099167104:192.168.202.128@tcp:250:kworker.0 00000008:00100000:0.0:1775768323.127793:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768323.127804:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022099167360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768323.127810:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768323.127813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768323.127814:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768323.127816:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5784 00000100:00080000:0.0:1775768323.127819:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768323.127821:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768323.127824:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768323.127831:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022099167488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768323.127836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768323.127838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768323.127840:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768323.127842:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5784 00000100:00080000:0.0:1775768323.127844:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768323.127846:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768323.127849:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768323.127854:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099167616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775768323.127881:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768323.127942:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768323/real 1775768323] req@ffff8b2382dad880 x1862022099167104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768323.127968:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022099167104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768323.127979:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768323.127983:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768323.127987:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099167104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768323.127995:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099167232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768323.128017:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768323.128029:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768323/real 1775768323] req@ffff8b2382dac380 x1862022099167232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768323.128040:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099167232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768323.128048:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768323.128051:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768323.128054:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099167232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768323.128089:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099167360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768323.128104:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768323.128110:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768323/real 1775768323] req@ffff8b2382dac000 x1862022099167360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768323.128122:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022099167360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768323.128127:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768323.128129:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768323.128131:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099167360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768323.128134:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099167488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768323.128144:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768323.128149:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768323/real 1775768323] req@ffff8b2382daf100 x1862022099167488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768323.128157:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022099167488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768323.128164:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768323.128168:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768323.128170:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099167488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768323.128175:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099167616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768323.128194:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768323.128204:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768323/real 1775768323] req@ffff8b2382daca80 x1862022099167616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768323.128215:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099167616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768323.128230:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768323.128233:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768323.128236:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099167616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768324.023617:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768325.047728:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768326.071713:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768327.095627:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768328.119634:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768328.247562:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768328.247572:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768328.247576:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768328.247609:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5789 00000100:00080000:0.0:1775768328.247621:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768328.247628:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768328.247688:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099167744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768328.247720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768328.247723:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768328.247724:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768328.247727:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5789 00000100:00080000:0.0:1775768328.247730:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768328.247733:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768328.247742:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768328.247756:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099167872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768328.247764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768328.247767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768328.247768:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768328.247771:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5789 00000100:00080000:0.0:1775768328.247773:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768328.247776:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768328.247780:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768328.247781:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099167744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768328.247789:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099168000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768328.247801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768328.247803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768328.247804:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768328.247807:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5789 00000100:00080000:0.0:1775768328.247809:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768328.247811:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768328.247813:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768328.247821:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099168128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768328.247828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768328.247831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768328.247832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768328.247836:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5789 00000100:00080000:0.0:1775768328.247839:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768328.247842:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768328.247847:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768328.247857:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022099168256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775768328.247874:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768328.247936:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768328/real 1775768328] req@ffff8b2382daf480 x1862022099167744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768328.247952:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099167744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768328.247961:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768328.247964:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768328.247967:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099167744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768328.247972:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099167872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768328.247989:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768328.247998:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768328/real 1775768328] req@ffff8b2382dad500 x1862022099167872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768328.248005:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099167872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768328.248009:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768328.248012:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768328.248014:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099167872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768328.248036:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099168000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768328.248046:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768328.248052:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768328/real 1775768328] req@ffff8b2382daea00 x1862022099168000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768328.248058:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099168000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768328.248065:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768328.248067:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768328.248070:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099168000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768328.248075:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099168128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768328.248093:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768328.248103:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768328/real 1775768328] req@ffff8b2382daed80 x1862022099168128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768328.248114:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099168128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768328.248121:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768328.248123:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768328.248125:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099168128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768328.248129:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099168256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768328.248143:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768328.248149:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768328/real 1775768328] req@ffff8b2382dafb80 x1862022099168256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768328.248160:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022099168256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768328.248167:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768328.248170:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768328.248173:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099168256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768329.143622:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768330.167694:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768331.191693:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768332.215669:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768333.239672:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768333.367584:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768333.367597:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768333.367603:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768333.367615:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5794 00000100:00080000:1.0:1775768333.367629:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768333.367636:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768333.367676:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099168384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768333.367703:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768333.367706:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768333.367708:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768333.367711:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5794 00000100:00080000:1.0:1775768333.367713:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768333.367716:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768333.367724:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768333.367734:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099168512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768333.367742:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768333.367744:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768333.367746:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768333.367749:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5794 00000100:00080000:1.0:1775768333.367752:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768333.367757:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768333.367762:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099168384:192.168.202.128@tcp:250:kworker.0 00000008:00100000:1.0:1775768333.367763:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768333.367779:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099168640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768333.367789:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768333.367792:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768333.367794:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768333.367797:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5794 00000100:00080000:1.0:1775768333.367800:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768333.367803:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768333.367807:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768333.367816:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099168768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768333.367822:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768333.367824:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768333.367825:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768333.367827:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5794 00000100:00080000:1.0:1775768333.367832:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768333.367834:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768333.367836:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768333.367842:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099168896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775768333.367851:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768333.367898:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768333/real 1775768333] req@ffff8b23864ffb80 x1862022099168384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768333.367914:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099168384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768333.367922:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768333.367931:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768333.367936:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099168384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768333.367964:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099168512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768333.367991:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768333.368004:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768333/real 1775768333] req@ffff8b23864fe300 x1862022099168512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768333.368018:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099168512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768333.368027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768333.368030:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768333.368033:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099168512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768333.368040:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099168640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768333.368065:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768333.368075:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768333/real 1775768333] req@ffff8b23864ff100 x1862022099168640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768333.368086:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099168640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768333.368091:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768333.368093:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768333.368095:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099168640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768333.368100:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099168768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768333.368114:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768333.368120:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768333/real 1775768333] req@ffff8b23864fd500 x1862022099168768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768333.368126:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099168768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768333.368131:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768333.368133:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768333.368135:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099168768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768333.368137:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099168896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768333.368146:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768333.368151:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768333/real 1775768333] req@ffff8b23864fdc00 x1862022099168896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768333.368156:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099168896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768333.368161:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768333.368162:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768333.368164:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099168896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768334.263620:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768335.287663:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768336.311669:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768337.335663:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768338.361740:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768338.487592:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768338.487599:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768338.487603:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768338.487611:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5799 00000100:00080000:3.0:1775768338.487620:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768338.487625:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768338.487654:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099169024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768338.487680:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768338.487683:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768338.487684:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768338.487687:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5799 00000100:00080000:3.0:1775768338.487689:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768338.487691:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768338.487697:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768338.487705:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099169152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768338.487713:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768338.487715:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768338.487717:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775768338.487722:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099169024:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768338.487722:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5799 00000100:00080000:3.0:1775768338.487727:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768338.487732:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768338.487738:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768338.487749:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099169280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768338.487758:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768338.487762:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768338.487764:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768338.487768:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5799 00000100:00080000:3.0:1775768338.487772:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768338.487776:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768338.487780:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768338.487791:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768338.487792:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099169408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768338.487799:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768338.487801:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768338.487802:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768338.487805:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5799 00000100:00080000:3.0:1775768338.487808:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768338.487811:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768338.487814:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768338.487821:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099169536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768338.487830:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768338/real 1775768338] req@ffff8b23864f2680 x1862022099169024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768338.487847:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099169024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768338.487857:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768338.487869:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768338.487873:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099169024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768338.487895:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099169152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768338.487911:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768338.487919:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768338/real 1775768338] req@ffff8b23864f2d80 x1862022099169152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768338.487926:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099169152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768338.487932:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768338.487935:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768338.487938:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099169152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768338.487943:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099169280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768338.487961:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768338.487970:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768338/real 1775768338] req@ffff8b23864f1180 x1862022099169280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768338.487981:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099169280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768338.487989:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768338.487992:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768338.487994:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099169280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768338.488000:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099169408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768338.488020:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768338.488029:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768338/real 1775768338] req@ffff8b23864f3b80 x1862022099169408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768338.488039:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099169408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768338.488046:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768338.488048:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768338.488050:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099169408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768338.488054:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099169536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768338.488072:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768338.488081:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768338/real 1775768338] req@ffff8b23864f0380 x1862022099169536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768338.488091:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099169536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768338.488099:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768338.488101:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768338.488104:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099169536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768339.383686:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768340.407603:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768341.431654:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768342.455651:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768343.479690:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768343.607629:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768343.607641:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768343.607647:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768343.607658:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5805 00000100:00080000:1.0:1775768343.607674:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768343.607681:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768343.607733:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099169664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768343.607785:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768343.607790:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768343.607791:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768343.607795:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5805 00000100:00080000:1.0:1775768343.607799:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768343.607804:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768343.607816:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768343.607831:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099169792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768343.607842:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768343.607846:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768343.607849:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768343.607853:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5805 00000100:00080000:1.0:1775768343.607856:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768343.607861:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768343.607866:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768343.607882:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099169920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768343.607891:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768343.607895:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768343.607897:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768343.607901:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5805 00000100:00080000:1.0:1775768343.607905:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768343.607908:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768343.607913:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768343.607924:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099170048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768343.607932:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768343.607936:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768343.607937:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768343.607941:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5805 00000100:00080000:1.0:1775768343.607944:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768343.607948:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768343.607953:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768343.607964:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099170176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768343.608076:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099169664:192.168.202.128@tcp:250:kworker.0 00000800:00000100:0.0:1775768343.608157:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768343.608206:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768343/real 1775768343] req@ffff8b23864fea00 x1862022099169664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768343.608228:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099169664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768343.608239:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768343.608243:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768343.608246:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099169664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768343.608254:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099169792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768343.608277:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768343.608290:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768343/real 1775768343] req@ffff8b23864fd180 x1862022099169792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768343.608302:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099169792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768343.608310:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768343.608313:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768343.608316:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099169792:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768343.608322:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099169920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768343.608341:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768343.608352:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768343/real 1775768343] req@ffff8b23864fce00 x1862022099169920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768343.608370:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099169920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768343.608379:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768343.608382:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768343.608385:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099169920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768343.608391:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099170048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768343.608411:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768343.608421:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768343/real 1775768343] req@ffff8b23864fe680 x1862022099170048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768343.608433:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099170048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768343.608501:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768343.608506:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768343.608510:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099170048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768343.608550:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099170176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768343.608576:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768343.608589:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768343/real 1775768343] req@ffff8b23864fca80 x1862022099170176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768343.608602:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099170176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768343.608616:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768343.608620:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768343.608623:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099170176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768344.504517:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768345.528661:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768346.552256:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768347.575670:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768348.599623:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768348.727646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768348.727659:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768348.727664:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768348.727677:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5810 00000100:00080000:0.0:1775768348.727692:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768348.727701:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768348.727754:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac700 x1862022099170304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768348.727792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768348.727796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768348.727798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768348.727802:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5810 00000100:00080000:0.0:1775768348.727806:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768348.727810:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768348.727821:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768348.727838:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dace00 x1862022099170432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768348.727857:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775768348.727859:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099170304:192.168.202.128@tcp:250:kworker.0 00000100:00080000:0.0:1775768348.727861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768348.727865:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768348.727873:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5810 00000100:00080000:0.0:1775768348.727876:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768348.727881:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768348.727887:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768348.727901:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dadf80 x1862022099170560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768348.727911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768348.727914:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768348.727916:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768348.727921:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5810 00000100:00080000:0.0:1775768348.727925:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768348.727929:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768348.727934:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768348.727944:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dae680 x1862022099170688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768348.727953:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1775768348.727957:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775768348.727959:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768348.727961:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768348.727966:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5810 00000100:00080000:0.0:1775768348.727971:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768348.727975:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768348.727980:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768348.727992:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022099170816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768348.728033:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768348/real 1775768348] req@ffff8b2382dac700 x1862022099170304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768348.728054:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac700 x1862022099170304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768348.728066:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768348.728070:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768348.728074:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099170304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768348.728102:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099170432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768348.728123:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768348.728133:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768348/real 1775768348] req@ffff8b2382dace00 x1862022099170432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768348.728144:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dace00 x1862022099170432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768348.728151:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768348.728154:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768348.728157:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099170432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768348.728162:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099170560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768348.728177:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768348.728185:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768348/real 1775768348] req@ffff8b2382dadf80 x1862022099170560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768348.728195:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dadf80 x1862022099170560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768348.728203:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768348.728206:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768348.728208:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099170560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768348.728213:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099170688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768348.728229:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768348.728249:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768348/real 1775768348] req@ffff8b2382dae680 x1862022099170688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768348.728270:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dae680 x1862022099170688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768348.728277:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768348.728280:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768348.728283:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099170688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768348.728288:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099170816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768348.728306:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768348.728314:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768348/real 1775768348] req@ffff8b2382dafb80 x1862022099170816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768348.728325:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022099170816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768348.728332:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768348.728335:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768348.728337:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099170816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768349.623699:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768350.647683:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768351.671670:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768352.696270:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768353.719692:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768353.847601:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768353.847611:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768353.847616:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768353.847628:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5815 00000100:00080000:1.0:1775768353.847643:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768353.847651:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768353.847709:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099170944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768353.847741:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768353.847745:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768353.847747:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768353.847751:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5815 00000100:00080000:1.0:1775768353.847754:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768353.847759:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768353.847770:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768353.847783:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099171072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768353.847796:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1775768353.847799:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099170944:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775768353.847800:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768353.847803:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768353.847812:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5815 00000100:00080000:1.0:1775768353.847815:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768353.847819:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768353.847824:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768353.847839:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099171200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768353.847848:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768353.847852:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768353.847853:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768353.847858:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5815 00000100:00080000:1.0:1775768353.847861:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768353.847864:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768353.847868:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768353.847881:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099171328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775768353.847886:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775768353.847896:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768353.847899:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768353.847901:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768353.847906:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5815 00000100:00080000:1.0:1775768353.847913:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768353.847917:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768353.847922:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768353.847934:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099171456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768353.847942:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768353/real 1775768353] req@ffff8b23864fdf80 x1862022099170944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768353.847963:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099170944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768353.847975:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768353.847980:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768353.847984:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099170944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768353.848025:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099171072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768353.848047:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768353.848058:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768353/real 1775768353] req@ffff8b23864ff800 x1862022099171072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768353.848071:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099171072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768353.848079:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768353.848082:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768353.848085:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099171072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768353.848091:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099171200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768353.848106:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768353.848115:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768353/real 1775768353] req@ffff8b23864fed80 x1862022099171200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768353.848132:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099171200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768353.848141:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768353.848144:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768353.848147:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099171200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768353.848153:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099171328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768353.848172:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768353.848194:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768353/real 1775768353] req@ffff8b23864fc000 x1862022099171328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768353.848206:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099171328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768353.848213:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768353.848217:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768353.848220:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099171328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768353.848225:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099171456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768353.848241:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768353.848249:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768353/real 1775768353] req@ffff8b23864fc380 x1862022099171456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768353.848259:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099171456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768353.848273:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768353.848276:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768353.848279:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099171456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768354.743688:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768355.767566:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768356.791694:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768357.815691:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768358.839680:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768358.967596:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768358.967611:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768358.967616:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768358.967629:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5820 00000100:00080000:3.0:1775768358.967643:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768358.967651:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768358.967699:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099171584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768358.967739:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768358.967742:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768358.967744:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768358.967746:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5820 00000100:00080000:3.0:1775768358.967750:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768358.967753:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768358.967762:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768358.967772:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099171712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768358.967780:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768358.967783:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768358.967785:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768358.967787:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5820 00000100:00080000:3.0:1775768358.967789:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768358.967792:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768358.967794:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768358.967800:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099171840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768358.967804:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768358.967806:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768358.967807:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768358.967811:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5820 00000100:00080000:3.0:1775768358.967813:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768358.967815:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768358.967817:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768358.967824:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099171968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768358.967828:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768358.967830:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768358.967831:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768358.967833:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5820 00000100:00080000:3.0:1775768358.967835:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768358.967837:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768358.967840:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768358.967846:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099172096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768358.967885:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099171584:192.168.202.128@tcp:250:kworker.0 00000800:00000100:0.0:1775768358.967961:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768358.968018:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768358/real 1775768358] req@ffff8b23864f3480 x1862022099171584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768358.968038:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099171584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768358.968049:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768358.968054:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768358.968058:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099171584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768358.968067:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099171712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768358.968091:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768358.968106:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768358/real 1775768358] req@ffff8b23864f2a00 x1862022099171712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768358.968127:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099171712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768358.968140:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768358.968143:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768358.968146:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099171712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768358.968152:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099171840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768358.968169:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768358.968178:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768358/real 1775768358] req@ffff8b23864f0000 x1862022099171840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768358.968190:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099171840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768358.968197:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768358.968201:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768358.968204:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099171840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768358.968209:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099171968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768358.968223:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768358.968231:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768358/real 1775768358] req@ffff8b23864f0380 x1862022099171968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768358.968241:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099171968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768358.968255:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768358.968258:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768358.968260:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099171968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768358.968266:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099172096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768358.968278:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768358.968284:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768358/real 1775768358] req@ffff8b23864f3b80 x1862022099172096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768358.968306:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099172096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768358.968315:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768358.968318:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768358.968321:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099172096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768359.863714:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768360.887694:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768361.911725:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768362.935732:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768363.959638:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768364.087608:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768364.087618:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768364.087623:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768364.087635:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5825 00000100:00080000:1.0:1775768364.087649:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768364.087657:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768364.087715:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099172224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768364.087750:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768364.087754:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768364.087755:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768364.087758:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5825 00000100:00080000:1.0:1775768364.087762:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768364.087765:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768364.087774:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768364.087784:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099172352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768364.087793:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768364.087799:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768364.087801:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768364.087805:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5825 00000100:00080000:1.0:1775768364.087808:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768364.087812:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768364.087817:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768364.087828:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099172480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768364.087837:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768364.087841:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768364.087843:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768364.087847:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5825 00000100:00080000:1.0:1775768364.087850:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768364.087853:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768364.087858:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768364.087866:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099172608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768364.087871:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768364.087874:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768364.087875:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768364.087877:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5825 00000100:00080000:1.0:1775768364.087879:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768364.087881:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768364.087885:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768364.087894:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099172736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768364.087996:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099172224:192.168.202.128@tcp:250:kworker.0 00000800:00000100:0.0:1775768364.088083:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768364.088145:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768364/real 1775768364] req@ffff8b23864fc380 x1862022099172224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768364.088162:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099172224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768364.088171:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768364.088175:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768364.088179:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099172224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768364.088185:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099172352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768364.088206:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768364.088218:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768364/real 1775768364] req@ffff8b23864fc000 x1862022099172352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768364.088228:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099172352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768364.088233:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768364.088235:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768364.088237:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099172352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768364.088252:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099172480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768364.088276:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768364.088287:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768364/real 1775768364] req@ffff8b23864fed80 x1862022099172480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768364.088298:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099172480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768364.088305:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768364.088308:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768364.088311:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099172480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768364.088316:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099172608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768364.088335:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768364.088345:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768364/real 1775768364] req@ffff8b23864ff800 x1862022099172608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768364.088356:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099172608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768364.088364:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768364.088367:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768364.088370:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099172608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768364.088376:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099172736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768364.088394:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768364.088404:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768364/real 1775768364] req@ffff8b23864fdf80 x1862022099172736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768364.088415:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099172736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768364.088423:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768364.088426:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768364.088429:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099172736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768364.983694:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768366.007684:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768367.031690:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768368.055604:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768369.079691:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768369.207623:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768369.207634:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768369.207639:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768369.207650:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5830 00000100:00080000:0.0:1775768369.207664:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768369.207672:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768369.207720:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099172864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768369.207756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768369.207759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768369.207760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768369.207764:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5830 00000100:00080000:0.0:1775768369.207770:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768369.207774:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768369.207782:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768369.207795:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099172992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768369.207807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768369.207811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768369.207813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768369.207818:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5830 00000100:00080000:0.0:1775768369.207821:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768369.207826:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768369.207832:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768369.207833:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099172864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768369.207846:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099173120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768369.207862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768369.207866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768369.207868:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768369.207872:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5830 00000100:00080000:0.0:1775768369.207876:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768369.207879:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768369.207885:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768369.207896:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099173248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768369.207904:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768369.207907:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768369.207908:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1775768369.207912:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775768369.207914:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5830 00000100:00080000:0.0:1775768369.207918:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768369.207921:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768369.207925:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768369.207934:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099173376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768369.207960:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768369/real 1775768369] req@ffff8b2382daed80 x1862022099172864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768369.207980:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099172864/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768369.207991:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768369.207996:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768369.208000:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099172864:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768369.208007:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099172992:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768369.208030:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768369.208042:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768369/real 1775768369] req@ffff8b2382daea00 x1862022099172992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768369.208055:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099172992/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768369.208069:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768369.208072:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768369.208076:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099172992:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768369.208102:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099173120:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768369.208118:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768369.208127:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768369/real 1775768369] req@ffff8b2382dad500 x1862022099173120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768369.208137:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099173120/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768369.208145:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768369.208148:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768369.208151:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099173120:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768369.208156:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099173248:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768369.208174:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768369.208184:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768369/real 1775768369] req@ffff8b2382daf480 x1862022099173248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768369.208210:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099173248/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768369.208218:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768369.208221:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768369.208224:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099173248:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768369.208230:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099173376:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768369.208248:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768369.208257:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768369/real 1775768369] req@ffff8b2382daca80 x1862022099173376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768369.208267:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099173376/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768369.208275:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768369.208278:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768369.208281:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099173376:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768370.103823:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768371.127680:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768372.152260:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768373.175649:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768374.199708:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768374.327593:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768374.327603:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768374.327608:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768374.327618:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5835 00000100:00080000:0.0:1775768374.327628:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768374.327633:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768374.327665:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf100 x1862022099173504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768374.327695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768374.327699:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768374.327701:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768374.327716:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5835 00000100:00080000:0.0:1775768374.327721:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768374.327726:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768374.327736:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768374.327747:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099173504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768374.327749:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac000 x1862022099173632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768374.327764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768374.327768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768374.327770:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768374.327774:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5835 00000100:00080000:0.0:1775768374.327778:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768374.327782:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768374.327787:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768374.327799:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dac380 x1862022099173760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1775768374.327810:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:0.0:1775768374.327831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768374.327834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768374.327837:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768374.327845:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5835 00000100:00080000:0.0:1775768374.327849:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768374.327853:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768374.327859:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768374.327867:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768374/real 1775768374] req@ffff8b2382daf100 x1862022099173504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768374.327871:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad880 x1862022099173888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768374.327893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768374.327896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768374.327898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768374.327901:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5835 00000100:00080000:0.0:1775768374.327904:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768374.327907:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768374.327911:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768374.327913:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf100 x1862022099173504/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768374.327925:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daca80 x1862022099174016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768374.327925:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768374.327934:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768374.327943:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099173504:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768374.327970:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099173632:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768374.327995:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768374.328010:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768374/real 1775768374] req@ffff8b2382dac000 x1862022099173632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768374.328040:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac000 x1862022099173632/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768374.328047:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768374.328050:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768374.328053:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099173632:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768374.328059:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099173760:192.168.202.128@tcp:38:kworker.0 00000800:00000100:3.0:1775768374.328078:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768374.328087:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768374/real 1775768374] req@ffff8b2382dac380 x1862022099173760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768374.328096:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dac380 x1862022099173760/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768374.328103:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768374.328105:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768374.328108:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099173760:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768374.328113:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099173888:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768374.328131:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768374.328154:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768374/real 1775768374] req@ffff8b2382dad880 x1862022099173888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768374.328167:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad880 x1862022099173888/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768374.328173:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768374.328176:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768374.328179:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099173888:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768374.328185:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099174016:192.168.202.128@tcp:8:kworker.0 00000800:00000100:3.0:1775768374.328205:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:3.0:1775768374.328214:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768374/real 1775768374] req@ffff8b2382daca80 x1862022099174016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768374.328224:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daca80 x1862022099174016/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768374.328231:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768374.328235:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768374.328238:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099174016:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768375.223664:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768376.247645:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768377.271755:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768378.296370:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768379.320066:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768379.447526:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768379.447535:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768379.447539:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768379.447548:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5840 00000100:00080000:0.0:1775768379.447559:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768379.447566:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768379.447600:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daf480 x1862022099174144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768379.447631:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768379.447634:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768379.447635:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768379.447638:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5840 00000100:00080000:0.0:1775768379.447641:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768379.447643:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768379.447650:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768379.447668:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dad500 x1862022099174272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768379.447679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768379.447681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768379.447683:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768379.447687:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5840 00000100:00080000:0.0:1775768379.447690:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768379.447694:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768379.447699:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768379.447709:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daea00 x1862022099174400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768379.447724:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768379.447727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768379.447728:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768379.447731:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5840 00000100:00080000:0.0:1775768379.447733:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768379.447736:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768379.447739:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768379.447746:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382daed80 x1862022099174528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768379.447751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768379.447753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1775768379.447754:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768379.447755:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5840 00000100:00080000:0.0:1775768379.447757:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768379.447758:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:0.0:1775768379.447761:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768379.447769:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b2382dafb80 x1862022099174656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1775768379.447848:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099174144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768379.447933:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099174272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768379.447958:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099174400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768379.447974:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099174528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768379.447990:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099174656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768380.343655:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768381.368547:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768382.391611:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775768382.519754:0:2364:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:2.0:1775768382.519811:0:2364:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775768382.519826:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775768382.519866:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775768382.519874:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775768382.519886:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:2.0:1775768382.519893:0:2364:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:3.0:1775768382.520052:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768379/real 1775768382] req@ffff8b2382daf480 x1862022099174144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768382.520086:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daf480 x1862022099174144/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768382.520096:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768382.520101:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768382.520105:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099174144:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768382.520113:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768379/real 1775768382] req@ffff8b2382dad500 x1862022099174272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768382.520124:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dad500 x1862022099174272/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768382.520131:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768382.520134:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768382.520137:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099174272:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768382.520143:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768379/real 1775768382] req@ffff8b2382daea00 x1862022099174400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768382.520153:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daea00 x1862022099174400/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768382.520166:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768382.520170:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768382.520173:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099174400:192.168.202.128@tcp:38:kworker.0 00000100:00100000:3.0:1775768382.520178:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768379/real 1775768382] req@ffff8b2382daed80 x1862022099174528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768382.520189:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382daed80 x1862022099174528/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768382.520195:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768382.520199:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768382.520201:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099174528:192.168.202.128@tcp:8:kworker.0 00000100:00100000:3.0:1775768382.520207:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768379/real 1775768382] req@ffff8b2382dafb80 x1862022099174656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768382.520217:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b2382dafb80 x1862022099174656/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768382.520225:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775768382.520228:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:3.0:1775768382.520231:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b2382dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099174656:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768383.415700:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768384.441324:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768384.567642:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768384.567651:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768384.567657:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768384.567667:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5845 00000100:00080000:3.0:1775768384.567679:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768384.567686:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768384.567812:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099174784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768384.567840:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768384.567843:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768384.567849:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768384.567853:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5845 00000100:00080000:3.0:1775768384.567857:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768384.567861:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768384.567873:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768384.567882:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099174784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768384.567883:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099174912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768384.567894:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768384.567897:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768384.567898:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768384.567902:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5845 00000100:00080000:3.0:1775768384.567905:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768384.567908:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768384.567912:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768384.567954:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768384.567997:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768384/real 1775768384] req@ffff8b23864f1180 x1862022099174784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768384.568017:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099174784/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768384.568027:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768384.568031:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768384.568034:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099174784:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768384.568037:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099175040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768384.568067:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768384.568069:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099174912:192.168.202.128@tcp:38:kworker.0 00000100:00080000:3.0:1775768384.568071:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768384.568073:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768384.568077:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5845 00000100:00080000:3.0:1775768384.568081:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768384.568084:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768384.568090:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768384.568091:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768384.568102:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768384/real 1775768384] req@ffff8b23864f2d80 x1862022099174912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768384.568102:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099175168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768384.568112:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768384.568113:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099174912/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768384.568115:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768384.568117:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768384.568120:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5845 00000100:00080000:0.0:1775768384.568121:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768384.568124:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00080000:3.0:1775768384.568124:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775768384.568127:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099174912:192.168.202.128@tcp:38:kworker.0 00000100:00080000:3.0:1775768384.568128:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768384.568133:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099175040:192.168.202.128@tcp:38:kworker.0 00000008:00100000:3.0:1775768384.568133:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768384.568141:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099175296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768384.568149:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768384.568157:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768384/real 1775768384] req@ffff8b23864f2680 x1862022099175040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768384.568167:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099175040/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768384.568181:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768384.568184:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768384.568187:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099175040:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768384.568200:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099175168:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768384.568228:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768384.568237:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768384/real 1775768384] req@ffff8b23864f1c00 x1862022099175168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768384.568246:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099175168/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768384.568253:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768384.568257:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768384.568260:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099175168:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768384.568265:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099175296:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768384.568283:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768384.568293:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768384/real 1775768384] req@ffff8b23864f0700 x1862022099175296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768384.568303:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099175296/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768384.568310:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768384.568313:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768384.568316:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099175296:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768385.463779:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768386.488494:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768387.513252:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768388.535675:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768389.559671:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768389.688158:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768389.688171:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768389.688178:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768389.688191:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5851 00000100:00080000:3.0:1775768389.688205:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768389.688214:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768389.688264:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099175424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768389.688297:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768389.688301:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768389.688303:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768389.688308:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5851 00000100:00080000:3.0:1775768389.688311:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768389.688314:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768389.688326:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768389.688342:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099175552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768389.688355:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768389.688359:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768389.688361:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775768389.688364:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099175424:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768389.688368:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5851 00000100:00080000:3.0:1775768389.688375:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768389.688380:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768389.688386:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768389.688399:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099175680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768389.688415:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768389.688418:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768389.688420:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768389.688424:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5851 00000100:00080000:3.0:1775768389.688427:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768389.688431:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768389.688436:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768389.688448:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099175808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768389.688495:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775768389.688500:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775768389.688501:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768389.688504:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768389.688509:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5851 00000100:00080000:3.0:1775768389.688514:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768389.688518:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768389.688524:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768389.688536:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099175936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768389.688556:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768389/real 1775768389] req@ffff8b23864f3100 x1862022099175424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768389.688576:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099175424/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768389.688587:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768389.688592:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768389.688597:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099175424:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768389.688624:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099175552:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768389.688647:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768389.688660:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768389/real 1775768389] req@ffff8b23864f0a80 x1862022099175552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768389.688672:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099175552/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768389.688680:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768389.688683:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768389.688686:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099175552:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768389.688694:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099175680:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768389.688713:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768389.688723:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768389/real 1775768389] req@ffff8b23864f1f80 x1862022099175680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768389.688734:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099175680/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768389.688741:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768389.688744:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768389.688747:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099175680:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768389.688752:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099175808:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768389.688774:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768389.688783:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768389/real 1775768389] req@ffff8b23864f0700 x1862022099175808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768389.688792:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099175808/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768389.688799:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768389.688802:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768389.688804:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099175808:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768389.688809:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099175936:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768389.688824:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768389.688831:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768389/real 1775768389] req@ffff8b23864f1c00 x1862022099175936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768389.688841:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099175936/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768389.688848:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768389.688851:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768389.688854:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099175936:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768390.583689:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768391.607886:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768392.631652:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768393.655696:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768394.679683:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768394.807653:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768394.807662:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768394.807665:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768394.807674:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5856 00000100:00080000:1.0:1775768394.807689:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768394.807696:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768394.807736:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099176064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768394.807771:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768394.807775:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768394.807776:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768394.807787:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5856 00000100:00080000:1.0:1775768394.807791:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768394.807795:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768394.807803:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768394.807817:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099176192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768394.807828:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768394.807831:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768394.807833:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768394.807837:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5856 00000100:00080000:1.0:1775768394.807841:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768394.807845:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768394.807850:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768394.807856:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099176064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:1.0:1775768394.807878:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099176320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768394.807890:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768394.807893:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768394.807895:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768394.807901:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5856 00000100:00080000:1.0:1775768394.807924:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768394.807928:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768394.807934:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768394.807938:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775768394.807966:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099176448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768394.807976:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768394.807979:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768394.807981:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768394.807986:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5856 00000100:00080000:1.0:1775768394.807990:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768394.808001:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768394.808007:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768394.808021:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768394/real 1775768394] req@ffff8b23864fca80 x1862022099176064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768394.808026:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099176576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768394.808077:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099176064/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768394.808091:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768394.808097:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768394.808102:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099176064:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768394.808110:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099176192:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768394.808133:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768394.808147:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768394/real 1775768394] req@ffff8b23864fe680 x1862022099176192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768394.808158:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099176192/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768394.808166:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768394.808169:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768394.808171:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099176192:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768394.808203:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099176320:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768394.808224:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768394.808234:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768394/real 1775768394] req@ffff8b23864fce00 x1862022099176320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768394.808243:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099176320/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768394.808251:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768394.808253:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768394.808256:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099176320:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768394.808261:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099176448:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768394.808280:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768394.808289:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768394/real 1775768394] req@ffff8b23864fd180 x1862022099176448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768394.808299:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099176448/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768394.808306:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768394.808309:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768394.808313:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099176448:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768394.808319:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099176576:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768394.808336:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768394.808344:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768394/real 1775768394] req@ffff8b23864fea00 x1862022099176576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768394.808351:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099176576/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768394.808356:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768394.808358:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768394.808360:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099176576:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768395.703608:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768396.727988:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768397.751602:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768398.775663:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768399.799688:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768399.927631:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768399.927644:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768399.927650:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768399.927663:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5861 00000100:00080000:3.0:1775768399.927677:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768399.927685:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768399.927745:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099176704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768399.927778:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768399.927781:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768399.927783:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768399.927787:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5861 00000100:00080000:3.0:1775768399.927791:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768399.927794:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768399.927805:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768399.927820:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099176832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768399.927831:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768399.927835:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768399.927838:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775768399.927839:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099176704:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768399.927844:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5861 00000100:00080000:3.0:1775768399.927848:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768399.927852:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768399.927858:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768399.927872:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099176960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768399.927880:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768399.927884:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768399.927886:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768399.927891:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5861 00000100:00080000:3.0:1775768399.927906:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768399.927911:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768399.927916:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768399.927928:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099177088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768399.927937:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768399.927943:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1775768399.927944:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775768399.927947:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768399.927952:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5861 00000100:00080000:3.0:1775768399.927956:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768399.927961:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768399.927966:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768399.927980:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099177216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768399.928004:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768399/real 1775768399] req@ffff8b23864f2680 x1862022099176704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768399.928026:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099176704/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768399.928038:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768399.928043:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768399.928047:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099176704:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768399.928081:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099176832:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768399.928103:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768399.928116:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768399/real 1775768399] req@ffff8b23864f2d80 x1862022099176832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768399.928128:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099176832/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768399.928133:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768399.928135:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768399.928138:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099176832:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768399.928142:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099176960:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768399.928157:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768399.928166:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768399/real 1775768399] req@ffff8b23864f1180 x1862022099176960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768399.928176:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099176960/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768399.928184:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768399.928187:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768399.928190:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099176960:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768399.928196:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099177088:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768399.928216:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768399.928226:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768399/real 1775768399] req@ffff8b23864f3b80 x1862022099177088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768399.928237:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099177088/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768399.928245:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768399.928248:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768399.928251:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099177088:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768399.928257:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099177216:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768399.928276:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768399.928285:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768399/real 1775768399] req@ffff8b23864f0380 x1862022099177216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768399.928296:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099177216/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768399.928304:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768399.928307:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768399.928310:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099177216:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768400.823673:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768401.847665:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768402.871683:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768403.897120:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768404.919754:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768405.047710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768405.047721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768405.047726:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768405.047740:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5866 00000100:00080000:2.0:1775768405.047755:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768405.047762:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768405.047808:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099177344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768405.047842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768405.047860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768405.047862:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768405.047866:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5866 00000100:00080000:2.0:1775768405.047872:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768405.047877:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768405.047889:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768405.047897:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099177344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:2.0:1775768405.047909:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099177472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768405.047923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768405.047927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768405.047929:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768405.047933:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5866 00000100:00080000:2.0:1775768405.047938:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768405.047943:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768405.047948:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768405.047961:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099177600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768405.047970:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768405.047974:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768405.047975:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768405.047979:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5866 00000100:00080000:2.0:1775768405.047983:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775768405.047987:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:2.0:1775768405.047991:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768405.047996:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768405.048012:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099177728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768405.048029:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768405.048032:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768405.048034:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768405.048038:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5866 00000100:00080000:2.0:1775768405.048042:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768405.048046:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768405.048051:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768405.048055:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768405/real 1775768405] req@ffff8b23b9ea7480 x1862022099177344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768405.048065:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099177856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768405.048081:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099177344/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768405.048095:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768405.048100:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768405.048103:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099177344:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768405.048127:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099177472:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768405.048145:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768405.048155:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768405/real 1775768405] req@ffff8b23b9ea6300 x1862022099177472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768405.048163:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099177472/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768405.048170:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768405.048174:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768405.048177:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099177472:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768405.048182:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099177600:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768405.048200:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768405.048211:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768405/real 1775768405] req@ffff8b23b9ea7100 x1862022099177600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768405.048222:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099177600/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768405.048229:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768405.048232:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768405.048235:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099177600:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768405.048242:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099177728:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768405.048263:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768405.048273:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768405/real 1775768405] req@ffff8b23b9ea4380 x1862022099177728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768405.048284:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099177728/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768405.048292:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768405.048294:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768405.048297:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099177728:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768405.048303:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099177856:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768405.048324:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768405.048333:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768405/real 1775768405] req@ffff8b23b9ea4700 x1862022099177856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768405.048345:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099177856/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768405.048352:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768405.048355:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768405.048357:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099177856:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768405.943734:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768406.967638:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768407.991659:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768409.015645:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768410.039711:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768410.167616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768410.167627:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768410.167633:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768410.167646:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5871 00000100:00080000:1.0:1775768410.167661:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768410.167669:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768410.167719:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099177984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768410.167750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768410.167755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768410.167757:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768410.167761:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5871 00000100:00080000:1.0:1775768410.167765:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768410.167769:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768410.167781:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768410.167792:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099178112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768410.167814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768410.167816:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768410.167818:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768410.167825:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5871 00000100:00100000:0.0:1775768410.167827:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099177984:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775768410.167829:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768410.167837:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768410.167855:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768410.167872:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099178240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768410.167880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768410.167883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768410.167884:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768410.167887:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5871 00000100:00080000:1.0:1775768410.167893:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768410.167896:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768410.167900:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768410.167907:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775768410.167920:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099178368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768410.167930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768410.167934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768410.167936:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768410.167939:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5871 00000100:00080000:1.0:1775768410.167943:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768410.167947:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768410.167953:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768410.167959:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768410/real 1775768410] req@ffff8b23864fdc00 x1862022099177984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768410.167971:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099178496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768410.167982:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099177984/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768410.167994:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768410.167999:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768410.168004:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099177984:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768410.168013:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099178112:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768410.168036:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768410.168048:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768410/real 1775768410] req@ffff8b23864fd500 x1862022099178112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768410.168059:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099178112/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768410.168067:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768410.168070:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768410.168073:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099178112:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768410.168102:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099178240:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768410.168121:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768410.168131:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768410/real 1775768410] req@ffff8b23864ff100 x1862022099178240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768410.168148:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099178240/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768410.168156:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768410.168159:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768410.168162:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099178240:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768410.168167:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099178368:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768410.168186:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768410.168196:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768410/real 1775768410] req@ffff8b23864fe300 x1862022099178368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768410.168206:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099178368/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768410.168214:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768410.168217:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768410.168220:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099178368:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768410.168226:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099178496:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768410.168243:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768410.168265:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768410/real 1775768410] req@ffff8b23864ffb80 x1862022099178496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768410.168276:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099178496/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768410.168289:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768410.168293:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768410.168295:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099178496:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768411.063642:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768412.087580:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768413.111627:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768414.135634:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768415.159716:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768415.287560:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768415.287569:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768415.287573:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768415.287582:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5876 00000100:00080000:1.0:1775768415.287594:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768415.287600:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768415.287635:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099178624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768415.287665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768415.287669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768415.287671:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768415.287674:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5876 00000100:00080000:1.0:1775768415.287677:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768415.287681:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768415.287689:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768415.287701:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099178752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768415.287711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768415.287715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768415.287716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768415.287722:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5876 00000100:00100000:0.0:1775768415.287723:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099178624:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775768415.287725:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768415.287731:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768415.287738:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768415.287771:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099178880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768415.287788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768415.287792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768415.287793:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768415.287797:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5876 00000100:00080000:1.0:1775768415.287802:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768415.287806:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000800:00000100:0.0:1775768415.287811:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000008:00100000:1.0:1775768415.287814:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768415.287828:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099179008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768415.287837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768415.287840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768415.287842:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768415.287845:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5876 00000100:00080000:1.0:1775768415.287849:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768415.287855:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768415.287856:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768415/real 1775768415] req@ffff8b23864ffb80 x1862022099178624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775768415.287862:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768415.287881:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099179136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768415.287883:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099178624/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768415.287910:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768415.287915:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768415.287919:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099178624:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768415.287927:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099178752:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768415.287950:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768415.287960:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768415/real 1775768415] req@ffff8b23864fe300 x1862022099178752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768415.287972:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099178752/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768415.287978:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768415.287982:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768415.287985:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099178752:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768415.288008:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099178880:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768415.288027:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768415.288036:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768415/real 1775768415] req@ffff8b23864ff100 x1862022099178880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768415.288047:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099178880/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768415.288055:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768415.288058:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768415.288061:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099178880:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768415.288066:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099179008:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768415.288092:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768415.288101:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768415/real 1775768415] req@ffff8b23864fd500 x1862022099179008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768415.288111:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099179008/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768415.288118:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768415.288121:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768415.288124:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099179008:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768415.288130:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099179136:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768415.288147:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768415.288156:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768415/real 1775768415] req@ffff8b23864fdc00 x1862022099179136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768415.288166:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099179136/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768415.288172:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768415.288175:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768415.288178:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099179136:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768416.183670:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768417.207711:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768418.231673:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768419.255640:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768420.279743:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768420.407613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768420.407624:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768420.407629:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768420.407643:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5881 00000100:00080000:1.0:1775768420.407668:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768420.407680:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768420.407753:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099179264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768420.407786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768420.407790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768420.407792:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768420.407796:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5881 00000100:00080000:1.0:1775768420.407800:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768420.407805:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768420.407817:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768420.407833:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099179392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768420.407846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768420.407852:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099179264:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775768420.407852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768420.407855:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768420.407863:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5881 00000100:00080000:1.0:1775768420.407867:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768420.407871:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768420.407876:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768420.407896:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099179520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768420.407909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768420.407914:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768420.407916:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768420.407920:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5881 00000100:00080000:1.0:1775768420.407924:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768420.407930:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768420.407935:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768420.407941:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775768420.407950:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099179648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768420.407959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768420.407963:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768420.407965:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768420.407970:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5881 00000100:00080000:1.0:1775768420.407974:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768420.407978:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768420.407983:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768420.408001:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768420/real 1775768420] req@ffff8b23864fea00 x1862022099179264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768420.408002:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099179776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768420.408042:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099179264/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768420.408054:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768420.408059:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768420.408062:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099179264:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768420.408086:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099179392:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768420.408104:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768420.408114:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768420/real 1775768420] req@ffff8b23864fd180 x1862022099179392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768420.408122:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099179392/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768420.408127:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768420.408129:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768420.408131:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099179392:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768420.408134:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099179520:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768420.408146:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768420.408154:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768420/real 1775768420] req@ffff8b23864fce00 x1862022099179520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768420.408165:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099179520/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768420.408172:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768420.408175:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768420.408178:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099179520:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768420.408184:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099179648:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768420.408204:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768420.408214:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768420/real 1775768420] req@ffff8b23864fe680 x1862022099179648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768420.408225:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099179648/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768420.408233:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768420.408236:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768420.408238:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099179648:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768420.408245:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099179776:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768420.408264:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768420.408273:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768420/real 1775768420] req@ffff8b23864fca80 x1862022099179776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768420.408281:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099179776/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768420.408286:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768420.408288:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768420.408290:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099179776:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768421.303738:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768422.327665:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768423.351701:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768424.375703:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768425.399617:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768425.527633:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768425.527643:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768425.527673:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768425.527686:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5886 00000100:00080000:2.0:1775768425.527701:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768425.527709:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768425.527764:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4e00 x1862022099179904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768425.527797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768425.527802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768425.527804:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768425.527808:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5886 00000100:00080000:2.0:1775768425.527812:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768425.527815:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768425.527827:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768425.527842:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4700 x1862022099180032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768425.527854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768425.527859:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768425.527860:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768425.527866:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5886 00000100:00100000:0.0:1775768425.527868:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099179904:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775768425.527870:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768425.527878:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768425.527884:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768425.527897:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4380 x1862022099180160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768425.527907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768425.527911:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768425.527912:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768425.527917:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5886 00000100:00080000:2.0:1775768425.527920:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768425.527924:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768425.527930:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768425.527940:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775768425.527941:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7100 x1862022099180288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768425.527949:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768425.527952:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768425.527953:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768425.527956:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5886 00000100:00080000:2.0:1775768425.527959:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768425.527962:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768425.527966:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768425.527974:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6300 x1862022099180416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768425.527997:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768425/real 1775768425] req@ffff8b23b9ea4e00 x1862022099179904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768425.528013:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4e00 x1862022099179904/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768425.528022:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768425.528028:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768425.528031:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099179904:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768425.528059:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099180032:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768425.528080:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768425.528089:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768425/real 1775768425] req@ffff8b23b9ea4700 x1862022099180032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768425.528097:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4700 x1862022099180032/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768425.528102:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768425.528104:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768425.528106:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099180032:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768425.528110:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099180160:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768425.528128:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768425.528139:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768425/real 1775768425] req@ffff8b23b9ea4380 x1862022099180160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768425.528150:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4380 x1862022099180160/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768425.528157:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768425.528161:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768425.528165:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099180160:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768425.528170:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099180288:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768425.528186:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768425.528193:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768425/real 1775768425] req@ffff8b23b9ea7100 x1862022099180288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768425.528200:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7100 x1862022099180288/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768425.528205:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768425.528207:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768425.528210:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099180288:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768425.528215:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099180416:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768425.528235:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768425.528245:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768425/real 1775768425] req@ffff8b23b9ea6300 x1862022099180416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768425.528256:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6300 x1862022099180416/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768425.528263:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768425.528266:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768425.528269:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099180416:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768426.423695:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768427.447786:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768428.471680:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768429.495701:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768430.519758:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768430.647594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768430.647618:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768430.647623:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768430.647640:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5892 00000100:00080000:2.0:1775768430.647655:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768430.647663:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768430.647721:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7480 x1862022099180544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768430.647755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768430.647759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768430.647761:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768430.647765:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5892 00000100:00080000:2.0:1775768430.647770:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768430.647774:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768430.647786:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768430.647798:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5880 x1862022099180672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768430.647813:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099180544:192.168.202.128@tcp:250:kworker.0 00000100:00080000:2.0:1775768430.647819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768430.647824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768430.647825:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768430.647830:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5892 00000100:00080000:2.0:1775768430.647834:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768430.647838:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768430.647844:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768430.647857:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4000 x1862022099180800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768430.647866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768430.647870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768430.647871:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768430.647875:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5892 00000100:00080000:2.0:1775768430.647879:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768430.647883:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768430.647889:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768430.647900:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:2.0:1775768430.647900:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6a00 x1862022099180928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768430.647913:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768430.647916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768430.647917:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768430.647922:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5892 00000100:00080000:2.0:1775768430.647927:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768430.647931:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768430.647936:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768430.647946:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea4a80 x1862022099181056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768430.647967:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768430/real 1775768430] req@ffff8b23b9ea7480 x1862022099180544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768430.647994:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7480 x1862022099180544/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768430.648004:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768430.648017:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768430.648025:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099180544:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768430.648055:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099180672:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768430.648073:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768430.648082:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768430/real 1775768430] req@ffff8b23b9ea5880 x1862022099180672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768430.648090:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5880 x1862022099180672/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768430.648095:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768430.648098:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768430.648100:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099180672:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768430.648103:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099180800:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768430.648114:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768430.648119:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768430/real 1775768430] req@ffff8b23b9ea4000 x1862022099180800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768430.648126:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4000 x1862022099180800/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768430.648135:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768430.648137:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768430.648139:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099180800:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768430.648142:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099180928:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768430.648165:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768430.648175:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768430/real 1775768430] req@ffff8b23b9ea6a00 x1862022099180928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768430.648185:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6a00 x1862022099180928/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768430.648193:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768430.648196:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768430.648199:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099180928:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768430.648204:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099181056:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768430.648224:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768430.648234:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768430/real 1775768430] req@ffff8b23b9ea4a80 x1862022099181056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768430.648245:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea4a80 x1862022099181056/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768430.648253:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768430.648256:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768430.648259:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099181056:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768431.543718:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768432.568159:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768433.591617:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768434.615693:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768435.639788:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768435.767628:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768435.767638:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768435.767642:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768435.767651:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5897 00000100:00080000:3.0:1775768435.767661:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768435.767667:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768435.767703:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099181184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768435.767730:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768435.767733:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768435.767735:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768435.767739:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5897 00000100:00080000:3.0:1775768435.767742:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768435.767745:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768435.767754:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768435.767767:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099181312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768435.767778:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768435.767781:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099181184:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768435.767782:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768435.767790:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768435.767797:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5897 00000100:00080000:3.0:1775768435.767799:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768435.767802:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768435.767806:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768435.767818:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099181440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768435.767826:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768435.767830:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768435.767832:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768435.767836:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5897 00000100:00080000:3.0:1775768435.767839:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768435.767841:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768435.767846:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768435.767855:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099181568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768435.767859:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775768435.767868:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768435.767871:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768435.767873:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768435.767877:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5897 00000100:00080000:3.0:1775768435.767880:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768435.767883:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768435.767894:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768435.767904:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099181696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768435.767915:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768435/real 1775768435] req@ffff8b23864f0000 x1862022099181184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768435.767935:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099181184/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768435.767946:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768435.767951:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768435.767955:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099181184:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768435.767976:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099181312:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768435.768000:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768435.768012:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768435/real 1775768435] req@ffff8b23864f2a00 x1862022099181312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768435.768023:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099181312/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768435.768030:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768435.768033:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768435.768036:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099181312:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768435.768041:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099181440:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768435.768057:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768435.768065:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768435/real 1775768435] req@ffff8b23864f3480 x1862022099181440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768435.768076:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099181440/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768435.768083:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768435.768086:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768435.768089:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099181440:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768435.768094:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099181568:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768435.768111:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768435.768120:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768435/real 1775768435] req@ffff8b23864f0380 x1862022099181568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768435.768130:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099181568/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768435.768138:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768435.768141:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768435.768144:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099181568:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768435.768149:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099181696:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768435.768165:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768435.768173:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768435/real 1775768435] req@ffff8b23864f3b80 x1862022099181696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768435.768189:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099181696/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768435.768196:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768435.768199:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768435.768202:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099181696:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768436.664875:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768437.687634:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768438.711708:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768439.735674:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768440.759672:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768440.887605:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768440.887618:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768440.887623:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768440.887642:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5902 00000100:00080000:3.0:1775768440.887659:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768440.887666:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768440.887727:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099181824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768440.887761:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768440.887765:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768440.887767:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768440.887771:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5902 00000100:00080000:3.0:1775768440.887775:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768440.887779:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768440.887790:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768440.887804:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099181952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768440.887817:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768440.887821:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768440.887824:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775768440.887825:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099181824:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768440.887830:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5902 00000100:00080000:3.0:1775768440.887836:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768440.887841:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768440.887845:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768440.887857:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099182080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768440.887866:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768440.887869:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768440.887871:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768440.887875:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5902 00000100:00080000:3.0:1775768440.887880:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768440.887883:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768440.887888:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768440.887897:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099182208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768440.887906:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768440.887910:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1775768440.887912:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775768440.887912:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768440.887918:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5902 00000100:00080000:3.0:1775768440.887922:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768440.887926:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768440.887931:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768440.887941:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099182336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768440.887976:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768440/real 1775768440] req@ffff8b23864f1180 x1862022099181824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768440.887999:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099181824/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768440.888011:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768440.888016:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768440.888020:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099181824:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768440.888051:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099181952:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768440.888073:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768440.888085:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768440/real 1775768440] req@ffff8b23864f2d80 x1862022099181952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768440.888102:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099181952/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768440.888110:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768440.888113:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768440.888116:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099181952:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768440.888121:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099182080:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768440.888140:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768440.888150:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768440/real 1775768440] req@ffff8b23864f2680 x1862022099182080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768440.888162:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099182080/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768440.888170:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768440.888172:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768440.888175:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099182080:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768440.888181:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099182208:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768440.888200:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768440.888210:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768440/real 1775768440] req@ffff8b23864f1c00 x1862022099182208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768440.888220:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099182208/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768440.888232:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768440.888235:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768440.888238:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099182208:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768440.888243:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099182336:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768440.888260:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768440.888268:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768440/real 1775768440] req@ffff8b23864f0700 x1862022099182336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768440.888278:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099182336/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768440.888286:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768440.888289:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768440.888292:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099182336:192.168.202.128@tcp:8:kworker.0 00010000:00010000:3.0:1775768441.783996:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768442.807751:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768443.831730:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768444.855668:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768445.879809:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768446.007643:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768446.007658:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768446.007663:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768446.007676:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5907 00000100:00080000:2.0:1775768446.007688:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768446.007710:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768446.007755:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea7b80 x1862022099182464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768446.007790:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768446.007795:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768446.007796:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768446.007800:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5907 00000100:00080000:2.0:1775768446.007805:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768446.007809:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768446.007820:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768446.007832:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099182592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768446.007843:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768446.007849:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768446.007851:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768446.007855:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5907 00000100:00080000:2.0:1775768446.007859:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768446.007863:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768446.007867:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768446.007877:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099182720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768446.007884:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768446.007887:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768446.007889:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768446.007892:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5907 00000100:00080000:2.0:1775768446.007895:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768446.007898:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768446.007902:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768446.007912:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099182848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768446.007920:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768446.007924:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768446.007936:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768446.007940:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5907 00000100:00080000:2.0:1775768446.007944:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768446.007947:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768446.007953:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768446.007962:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099182976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768446.008509:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099182464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768446.008631:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099182592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768446.008662:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099182720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768446.008689:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099182848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768446.008715:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099182976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768446.903700:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768447.927752:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768448.951763:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775768449.079893:0:2365:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.128:988 00000400:00000100:1.0:1775768449.079949:0:2365:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.128@tcp at host 192.168.202.128 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775768449.079971:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775768449.080027:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775768449.080038:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775768449.080047:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000800:00000100:1.0:1775768449.080054:0:2365:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.128@tcp 00000100:00100000:0.0:1775768449.080640:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768446/real 1775768449] req@ffff8b23b9ea7b80 x1862022099182464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768449.080672:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea7b80 x1862022099182464/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768449.080685:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768449.080691:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768449.080696:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099182464:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768449.080705:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768446/real 1775768449] req@ffff8b23b9ea6680 x1862022099182592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768449.080720:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099182592/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768449.080727:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768449.080731:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768449.080735:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099182592:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768449.080743:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768446/real 1775768449] req@ffff8b23b9ea5f80 x1862022099182720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768449.080756:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099182720/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768449.080764:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768449.080768:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768449.080772:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099182720:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768449.080779:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768446/real 1775768449] req@ffff8b23b9ea6d80 x1862022099182848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768449.080790:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099182848/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768449.080797:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768449.080801:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768449.080804:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099182848:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768449.080811:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768446/real 1775768449] req@ffff8b23b9ea5c00 x1862022099182976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768449.080821:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099182976/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768449.080828:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768449.080832:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768449.080835:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099182976:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768449.975661:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768451.000555:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768451.127612:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768451.127623:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768451.127629:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768451.127640:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5912 00000100:00080000:3.0:1775768451.127654:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768451.127662:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768451.127716:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1f80 x1862022099183104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768451.127749:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768451.127753:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768451.127755:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768451.127759:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5912 00000100:00080000:3.0:1775768451.127763:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768451.127766:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768451.127777:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768451.127805:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0a80 x1862022099183232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768451.127814:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099183104:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768451.127841:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768451.127845:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768451.127847:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768451.127852:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5912 00000100:00080000:3.0:1775768451.127856:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768451.127861:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768451.127866:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768451.127880:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3100 x1862022099183360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768451.127889:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768451.127893:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768451.127895:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775768451.127899:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775768451.127903:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5912 00000100:00080000:3.0:1775768451.127908:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768451.127923:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768451.127929:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768451.127942:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0700 x1862022099183488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768451.127951:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768451/real 1775768451] req@ffff8b23864f1f80 x1862022099183104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768451.127953:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768451.127964:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768451.127968:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768451.127973:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5912 00000100:00080000:3.0:1775768451.127979:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775768451.127981:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1f80 x1862022099183104/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768451.127986:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768451.127996:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1775768451.128000:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768451.128006:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768451.128012:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099183104:192.168.202.128@tcp:250:kworker.0 00000100:00100000:3.0:1775768451.128012:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1c00 x1862022099183616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768451.128045:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099183232:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768451.128067:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768451.128077:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768451/real 1775768451] req@ffff8b23864f0a80 x1862022099183232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768451.128089:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0a80 x1862022099183232/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768451.128097:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768451.128100:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768451.128103:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099183232:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768451.128109:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099183360:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768451.128124:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768451.128132:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768451/real 1775768451] req@ffff8b23864f3100 x1862022099183360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768451.128142:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3100 x1862022099183360/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768451.128150:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768451.128153:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768451.128156:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099183360:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768451.128162:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099183488:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768451.128178:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768451.128186:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768451/real 1775768451] req@ffff8b23864f0700 x1862022099183488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768451.128197:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0700 x1862022099183488/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768451.128205:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768451.128208:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768451.128211:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099183488:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768451.128216:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099183616:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768451.128231:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768451.128239:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768451/real 1775768451] req@ffff8b23864f1c00 x1862022099183616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768451.128249:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1c00 x1862022099183616/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768451.128257:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768451.128260:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768451.128263:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099183616:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768452.023634:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768453.047720:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768454.071658:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768455.095789:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768456.120028:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768456.247617:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768456.247630:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768456.247636:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768456.247649:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5917 00000100:00080000:1.0:1775768456.247663:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768456.247679:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768456.247728:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099183744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768456.247763:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768456.247804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768456.247807:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768456.247812:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5917 00000100:00080000:1.0:1775768456.247816:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768456.247823:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:0.0:1775768456.247833:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099183744:192.168.202.128@tcp:250:kworker.0 00000008:00100000:1.0:1775768456.247835:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768456.247854:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099183872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768456.247869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768456.247874:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768456.247877:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768456.247881:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5917 00000100:00080000:1.0:1775768456.247908:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768456.247913:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768456.247918:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768456.247923:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775768456.247935:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099184000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768456.247946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768456.247950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768456.247953:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768456.247958:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5917 00000100:00080000:1.0:1775768456.247962:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768456.247966:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768456.247972:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768456.247980:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768456/real 1775768456] req@ffff8b23864fdf80 x1862022099183744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768456.247985:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099184128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768456.248009:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768456.248012:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768456.248014:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775768456.248020:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099183744/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768456.248020:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5917 00000100:00080000:1.0:1775768456.248027:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768456.248033:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00080000:0.0:1775768456.248034:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768456.248041:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000008:00100000:1.0:1775768456.248042:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768456.248045:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099183744:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768456.248070:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099183872:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768456.248087:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768456.248096:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768456/real 1775768456] req@ffff8b23864ff800 x1862022099183872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768456.248112:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099183872/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768456.248120:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768456.248122:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768456.248124:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099183872:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768456.248128:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099184000:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768456.248140:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768456.248146:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768456/real 1775768456] req@ffff8b23864fed80 x1862022099184000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768456.248163:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099184000/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768456.248172:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1775768456.248176:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099184256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768456.248179:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768456.248187:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099184000:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768456.248197:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099184128:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768456.248219:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768456.248230:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768456/real 1775768456] req@ffff8b23864fc000 x1862022099184128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768456.248238:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099184128/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768456.248243:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768456.248245:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768456.248247:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099184128:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768456.248258:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099184256:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768456.248270:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768456.248278:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768456/real 1775768456] req@ffff8b23864fc380 x1862022099184256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768456.248288:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099184256/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768456.248302:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768456.248305:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768456.248308:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099184256:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768457.143668:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768458.167641:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768459.191660:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768460.215700:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768461.241548:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768461.370508:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768461.370523:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768461.370528:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768461.370542:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5922 00000100:00080000:2.0:1775768461.370557:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768461.370565:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:2.0:1775768461.370614:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5500 x1862022099184384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768461.370658:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768461.370662:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768461.370664:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768461.370669:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5922 00000100:00080000:2.0:1775768461.370672:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768461.370676:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768461.370688:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768461.370703:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5c00 x1862022099184512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768461.370716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768461.370719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768461.370721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768461.370725:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5922 00000100:00080000:2.0:1775768461.370730:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768461.370734:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768461.370739:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768461.370750:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6d80 x1862022099184640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768461.370759:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768461.370763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768461.370765:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768461.370769:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5922 00000100:00080000:2.0:1775768461.370772:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768461.370775:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768461.370781:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768461.370790:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea5f80 x1862022099184768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768461.370799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768461.370802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1775768461.370804:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768461.370807:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5922 00000100:00080000:2.0:1775768461.370810:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768461.370814:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:2.0:1775768461.370818:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768461.370828:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23b9ea6680 x1862022099184896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768461.372127:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099184384:192.168.202.128@tcp:250:kworker.0 00000800:00000100:0.0:1775768461.372221:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768461.372371:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768461/real 1775768461] req@ffff8b23b9ea5500 x1862022099184384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768461.372392:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5500 x1862022099184384/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768461.372404:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768461.372409:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768461.372413:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099184384:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768461.372423:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099184512:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768461.372450:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768461.372504:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768461/real 1775768461] req@ffff8b23b9ea5c00 x1862022099184512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768461.372515:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5c00 x1862022099184512/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768461.372527:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768461.372531:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768461.372533:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099184512:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768461.372540:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099184640:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768461.372561:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768461.372571:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768461/real 1775768461] req@ffff8b23b9ea6d80 x1862022099184640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768461.372581:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6d80 x1862022099184640/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768461.372588:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768461.372591:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768461.372593:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099184640:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768461.372600:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099184768:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768461.372620:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768461.372629:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768461/real 1775768461] req@ffff8b23b9ea5f80 x1862022099184768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768461.372641:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea5f80 x1862022099184768/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768461.372649:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768461.372652:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768461.372655:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099184768:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768461.372661:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099184896:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768461.372682:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768461.372691:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768461/real 1775768461] req@ffff8b23b9ea6680 x1862022099184896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768461.372701:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23b9ea6680 x1862022099184896/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768461.372709:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768461.372712:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768461.372715:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23b9ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099184896:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768462.264449:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768463.287645:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768464.312247:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768465.337603:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768466.361514:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768466.488424:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768466.488433:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768466.488438:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768466.488447:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5927 00000100:00080000:1.0:1775768466.488477:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768466.488485:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768466.488518:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc380 x1862022099185024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768466.488548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768466.488552:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768466.488553:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768466.488557:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5927 00000100:00080000:1.0:1775768466.488560:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768466.488563:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768466.488571:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768466.488582:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fc000 x1862022099185152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768466.488598:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768466.488601:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768466.488602:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768466.488606:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5927 00000100:00080000:1.0:1775768466.488609:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768466.488612:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768466.488616:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768466.488625:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fed80 x1862022099185280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768466.488633:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768466.488636:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768466.488638:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768466.488641:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5927 00000100:00080000:1.0:1775768466.488643:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768466.488647:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768466.488651:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768466.488658:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff800 x1862022099185408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768466.488666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768466.488668:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768466.488670:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768466.488673:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5927 00000100:00080000:1.0:1775768466.488676:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768466.488679:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768466.488682:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768466.488690:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdf80 x1862022099185536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768466.488769:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099185024:192.168.202.128@tcp:250:kworker.0 00000800:00000100:0.0:1775768466.488823:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768466.488856:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768466/real 1775768466] req@ffff8b23864fc380 x1862022099185024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768466.488871:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc380 x1862022099185024/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768466.488880:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768466.488883:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768466.488886:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099185024:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768466.488892:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099185152:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768466.488910:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768466.488920:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768466/real 1775768466] req@ffff8b23864fc000 x1862022099185152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768466.488929:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fc000 x1862022099185152/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768466.488936:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768466.488939:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768466.488941:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099185152:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768466.488946:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099185280:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768466.488961:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768466.488968:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768466/real 1775768466] req@ffff8b23864fed80 x1862022099185280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768466.488977:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fed80 x1862022099185280/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768466.488984:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768466.488987:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768466.488989:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099185280:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768466.488994:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099185408:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768466.489009:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768466.489017:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768466/real 1775768466] req@ffff8b23864ff800 x1862022099185408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768466.489033:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff800 x1862022099185408/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768466.489039:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768466.489042:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768466.489045:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099185408:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768466.489050:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099185536:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768466.489065:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768466.489073:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768466/real 1775768466] req@ffff8b23864fdf80 x1862022099185536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768466.489083:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdf80 x1862022099185536/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768466.489090:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768466.489093:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768466.489095:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099185536:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768467.386404:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768468.407694:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768469.432104:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768470.455711:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768471.483317:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768471.607663:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768471.607674:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768471.607678:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768471.607686:0:10503:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5933 00000100:00080000:3.0:1775768471.607697:0:10503:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768471.607703:0:10503:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768471.607739:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2680 x1862022099185664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768471.607821:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099185664:192.168.202.128@tcp:250:kworker.0 00000800:00000100:0.0:1775768471.607878:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:3.0:1775768471.607907:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768471.607912:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768471.607914:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768471.607917:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5933 00000100:00080000:3.0:1775768471.607922:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768471.607926:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768471.607934:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768471.607948:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2d80 x1862022099185792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768471.608004:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768471/real 1775768471] req@ffff8b23864f2680 x1862022099185664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768471.608028:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2680 x1862022099185664/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768471.608037:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768471.608041:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768471.608045:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099185664:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768471.608067:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099185792:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768471.608089:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768471.608102:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768471/real 1775768471] req@ffff8b23864f2d80 x1862022099185792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768471.608113:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2d80 x1862022099185792/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768471.608121:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768471.608123:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768471.608126:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099185792:192.168.202.128@tcp:38:kworker.0 00000100:00080000:3.0:1775768471.609187:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768471.609194:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768471.609197:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768471.609229:0:10503:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5933 00000100:00080000:3.0:1775768471.609235:0:10503:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768471.609239:0:10503:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768471.609246:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768471.609262:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f1180 x1862022099185920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768471.609273:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768471.609276:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768471.609278:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768471.609282:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5933 00000100:00080000:3.0:1775768471.609286:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768471.609289:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768471.609294:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768471.609303:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099186048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768471.609312:0:10503:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768471.609315:0:10503:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768471.609317:0:10503:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768471.609319:0:10503:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5933 00000100:00080000:3.0:1775768471.609322:0:10503:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768471.609326:0:10503:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768471.609331:0:10503:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768471.609340:0:10503:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099186176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768471.609411:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099185920:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768471.609442:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768471.609487:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768471/real 1775768471] req@ffff8b23864f1180 x1862022099185920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768471.609502:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f1180 x1862022099185920/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768471.609516:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768471.609520:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768471.609523:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099185920:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768471.609528:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099186048:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768471.609549:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768471.609559:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768471/real 1775768471] req@ffff8b23864f3b80 x1862022099186048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768471.609570:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099186048/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768471.609577:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768471.609581:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768471.609584:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099186048:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768471.609591:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099186176:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768471.609611:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768471.609622:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768471/real 1775768471] req@ffff8b23864f0380 x1862022099186176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768471.609633:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099186176/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768471.609640:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768471.609643:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768471.609646:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099186176:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768472.507278:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768473.530770:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768474.551728:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768475.575813:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768476.599899:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768476.727891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768476.727903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768476.727908:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768476.727921:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5938 00000100:00080000:1.0:1775768476.727935:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768476.727943:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768476.727995:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fca80 x1862022099186304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768476.728029:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768476.728033:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768476.728035:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768476.728038:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5938 00000100:00080000:1.0:1775768476.728042:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768476.728045:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768476.728056:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768476.728075:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe680 x1862022099186432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768476.728088:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768476.728093:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1775768476.728094:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099186304:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775768476.728095:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768476.728104:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5938 00000100:00080000:1.0:1775768476.728108:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768476.728112:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768476.728117:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768476.728130:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fce00 x1862022099186560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768476.728139:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768476.728143:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768476.728145:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768476.728165:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5938 00000100:00080000:1.0:1775768476.728184:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768476.728189:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768476.728194:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768476.728200:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775768476.728210:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd180 x1862022099186688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768476.728221:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768476.728224:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768476.728226:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768476.728231:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5938 00000100:00080000:1.0:1775768476.728235:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768476.728239:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768476.728244:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768476.728255:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768476/real 1775768476] req@ffff8b23864fca80 x1862022099186304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768476.728258:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fea00 x1862022099186816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768476.728295:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fca80 x1862022099186304/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768476.728307:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768476.728312:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768476.728317:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099186304:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768476.728346:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099186432:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768476.728369:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768476.728385:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768476/real 1775768476] req@ffff8b23864fe680 x1862022099186432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768476.728396:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe680 x1862022099186432/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768476.728488:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768476.728493:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768476.728497:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099186432:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768476.728503:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099186560:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768476.728526:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768476.728547:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768476/real 1775768476] req@ffff8b23864fce00 x1862022099186560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768476.728559:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fce00 x1862022099186560/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768476.728567:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768476.728571:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768476.728573:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099186560:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768476.728579:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099186688:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768476.728601:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768476.728611:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768476/real 1775768476] req@ffff8b23864fd180 x1862022099186688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768476.728622:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd180 x1862022099186688/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768476.728630:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768476.728633:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768476.728636:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099186688:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768476.728651:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099186816:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768476.728672:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768476.728682:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768476/real 1775768476] req@ffff8b23864fea00 x1862022099186816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768476.728701:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fea00 x1862022099186816/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768476.728709:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768476.728712:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768476.728715:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099186816:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768477.624307:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768478.649222:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768479.671701:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768480.695606:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768481.719696:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768481.847593:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768481.847606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768481.847612:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768481.847626:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5943 00000100:00080000:1.0:1775768481.847640:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768481.847649:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768481.847698:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099186944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768481.847731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768481.847735:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768481.847737:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768481.847740:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5943 00000100:00080000:1.0:1775768481.847744:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768481.847748:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768481.847759:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768481.847773:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099187072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768481.847780:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099186944:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775768481.847789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768481.847793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768481.847804:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768481.847813:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5943 00000100:00080000:1.0:1775768481.847818:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768481.847822:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768481.847828:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768481.847843:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099187200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768481.847853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775768481.847860:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00080000:1.0:1775768481.847867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768481.847869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768481.847874:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5943 00000100:00080000:1.0:1775768481.847879:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768481.847883:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768481.847889:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768481.847906:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099187328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768481.847916:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768481.847919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768481.847921:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775768481.847926:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768481/real 1775768481] req@ffff8b23864fdc00 x1862022099186944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768481.847928:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5943 00000100:00080000:1.0:1775768481.847935:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768481.847943:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768481.847948:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768481.847961:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099187456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768481.847966:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099186944/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768481.847983:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768481.847989:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768481.847994:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099186944:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768481.848021:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099187072:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768481.848040:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768481.848049:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768481/real 1775768481] req@ffff8b23864fd500 x1862022099187072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768481.848061:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099187072/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768481.848066:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768481.848068:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768481.848070:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099187072:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768481.848073:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099187200:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768481.848083:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768481.848089:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768481/real 1775768481] req@ffff8b23864ff100 x1862022099187200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768481.848095:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099187200/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768481.848120:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768481.848123:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768481.848126:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099187200:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768481.848131:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099187328:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768481.848151:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768481.848160:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768481/real 1775768481] req@ffff8b23864fe300 x1862022099187328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768481.848171:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099187328/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768481.848185:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768481.848188:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768481.848191:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099187328:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768481.848196:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099187456:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768481.848215:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768481.848224:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768481/real 1775768481] req@ffff8b23864ffb80 x1862022099187456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768481.848235:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099187456/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768481.848243:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768481.848245:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768481.848247:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099187456:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768482.743700:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768483.768798:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768484.791685:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768485.815755:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768486.839683:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768486.967558:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768486.967571:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768486.967576:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768486.967589:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5948 00000100:00080000:1.0:1775768486.967603:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768486.967612:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:1.0:1775768486.967674:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ffb80 x1862022099187584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768486.967705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768486.967709:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768486.967711:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768486.967715:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5948 00000100:00080000:1.0:1775768486.967719:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768486.967723:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768486.967734:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768486.967748:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fe300 x1862022099187712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768486.967759:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099187584:192.168.202.128@tcp:250:kworker.0 00000100:00080000:1.0:1775768486.967759:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768486.967762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768486.967764:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768486.967767:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5948 00000100:00080000:1.0:1775768486.967771:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768486.967775:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768486.967779:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768486.967788:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864ff100 x1862022099187840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768486.967796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768486.967798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768486.967800:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768486.967804:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5948 00000100:00080000:1.0:1775768486.967824:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768486.967829:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768486.967833:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768486.967834:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:1.0:1775768486.967847:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fd500 x1862022099187968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768486.967855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768486.967857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1775768486.967859:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768486.967862:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5948 00000100:00080000:1.0:1775768486.967865:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768486.967868:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:1.0:1775768486.967872:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768486.967880:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768486/real 1775768486] req@ffff8b23864ffb80 x1862022099187584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768486.967881:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864fdc00 x1862022099188096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768486.967900:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ffb80 x1862022099187584/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768486.967910:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768486.967916:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768486.967920:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ffb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099187584:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768486.967963:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099187712:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768486.967986:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768486.967998:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768486/real 1775768486] req@ffff8b23864fe300 x1862022099187712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768486.968009:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fe300 x1862022099187712/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768486.968016:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768486.968020:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768486.968023:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099187712:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768486.968028:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099187840:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768486.968057:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768486.968068:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768486/real 1775768486] req@ffff8b23864ff100 x1862022099187840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768486.968075:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864ff100 x1862022099187840/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768486.968081:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768486.968083:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768486.968085:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864ff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099187840:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768486.968089:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099187968:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768486.968109:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768486.968119:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768486/real 1775768486] req@ffff8b23864fd500 x1862022099187968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768486.968130:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fd500 x1862022099187968/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768486.968144:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768486.968148:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768486.968154:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099187968:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768486.968160:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099188096:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768486.968190:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768486.968201:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768486/real 1775768486] req@ffff8b23864fdc00 x1862022099188096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768486.968212:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864fdc00 x1862022099188096/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768486.968220:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768486.968223:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768486.968226:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099188096:192.168.202.128@tcp:8:kworker.0 00010000:00010000:0.0:1775768487.863651:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768488.887602:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768489.911625:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768490.935605:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768491.959662:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768492.087599:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 24b54882-c0a1-43f2-954f-009e5a9afc9d->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768492.087609:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768492.087614:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768492.087624:0:36:0:(import.c:509:import_select_connection()) MGC192.168.202.128@tcp: connect to NID 192.168.202.128@tcp last attempt 5953 00000100:00080000:3.0:1775768492.087636:0:36:0:(import.c:588:import_select_connection()) MGC192.168.202.128@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768492.087642:0:36:0:(import.c:623:import_select_connection()) MGC192.168.202.128@tcp: import ffff8b2387d04800 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000100:00100000:3.0:1775768492.087689:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3480 x1862022099188224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768492.087711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 82781e18-e0e3-4f9f-9a2b-041bf0344414->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768492.087714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768492.087716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768492.087718:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: connect to NID 192.168.202.128@tcp last attempt 5953 00000100:00080000:3.0:1775768492.087720:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768492.087722:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b238a3bf000: import ffff8b23a00b1000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768492.087731:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768492.087741:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f2a00 x1862022099188352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768492.087749:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768492.087751:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768492.087752:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768492.087755:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5953 00000100:00080000:3.0:1775768492.087758:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775768492.087761:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099188224:192.168.202.128@tcp:250:kworker.0 00000100:00080000:3.0:1775768492.087763:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b2389398000: import ffff8b2384b86000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768492.087770:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768492.087777:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0000 x1862022099188480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768492.087783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768492.087786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768492.087787:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768492.087789:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5953 00000100:00080000:3.0:1775768492.087791:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768492.087793:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b2389398000: import ffff8b2384b87000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768492.087795:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768492.087803:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f0380 x1862022099188608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768492.087808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) fcf15866-4739-4595-8fe1-44abde7ee9d7->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768492.087810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1775768492.087811:0:36:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768492.087813:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: connect to NID 192.168.202.128@tcp last attempt 5953 00000100:00080000:3.0:1775768492.087815:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768492.087817:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b2389398000: import ffff8b2384b83000 using connection 192.168.202.128@tcp/192.168.202.128@tcp 00000008:00100000:3.0:1775768492.087820:0:36:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681677 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768492.087826:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b23864f3b80 x1862022099188736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768492.087839:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768492.087885:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768492/real 1775768492] req@ffff8b23864f3480 x1862022099188224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768492.087900:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3480 x1862022099188224/t0(0) o250->MGC192.168.202.128@tcp@192.168.202.128@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768492.087908:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2387d04800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768492.087911:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768492.087914:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:24b54882-c0a1-43f2-954f-009e5a9afc9d:2375:1862022099188224:192.168.202.128@tcp:250:kworker.0 00000100:00100000:0.0:1775768492.087920:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099188352:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768492.087936:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768492.087948:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768492/real 1775768492] req@ffff8b23864f2a00 x1862022099188352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768492.087960:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f2a00 x1862022099188352/t0(0) o38->lustre-MDT0000-mdc-ffff8b238a3bf000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768492.087967:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b23a00b1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768492.087970:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768492.087974:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:82781e18-e0e3-4f9f-9a2b-041bf0344414:2375:1862022099188352:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768492.088036:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099188480:192.168.202.128@tcp:38:kworker.0 00000800:00000100:0.0:1775768492.088057:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768492.088067:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768492/real 1775768492] req@ffff8b23864f0000 x1862022099188480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768492.088078:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0000 x1862022099188480/t0(0) o38->lustre-MDT0000-mdc-ffff8b2389398000@192.168.202.128@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768492.088092:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b86000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768492.088095:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768492.088097:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099188480:192.168.202.128@tcp:38:kworker.0 00000100:00100000:0.0:1775768492.088103:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099188608:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768492.088119:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768492.088127:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768492/real 1775768492] req@ffff8b23864f0380 x1862022099188608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768492.088138:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f0380 x1862022099188608/t0(0) o8->lustre-OST0000-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768492.088145:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b87000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768492.088148:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768492.088151:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099188608:192.168.202.128@tcp:8:kworker.0 00000100:00100000:0.0:1775768492.088157:0:2375:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099188736:192.168.202.128@tcp:8:kworker.0 00000800:00000100:0.0:1775768492.088177:0:2375:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.128@tcp 00000100:00100000:0.0:1775768492.088187:0:2375:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768492/real 1775768492] req@ffff8b23864f3b80 x1862022099188736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768492.088198:0:2375:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b23864f3b80 x1862022099188736/t0(0) o8->lustre-OST0001-osc-ffff8b2389398000@192.168.202.128@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768492.088205:0:2375:0:(import.c:65:import_set_state_nolock()) ffff8b2384b83000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768492.088208:0:2375:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.202.128@tcp failed (-110) 00000100:00100000:0.0:1775768492.088212:0:2375:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff8b23864f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fcf15866-4739-4595-8fe1-44abde7ee9d7:2375:1862022099188736:192.168.202.128@tcp:8:kworker.0 00010000:00010000:1.0:1775768492.983625:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768494.007574:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768495.031640:0:46149:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768496.055603:0:46150:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 73977 lines, 73977 kept, 0 dropped, 0 bad.